Repository: stac-utils/pgstac Branch: main Commit: 19df4cc17995 Files: 274 Total size: 9.3 MB Directory structure: gitextract_odjuxhd0/ ├── .devcontainer/ │ └── devcontainer.json ├── .dockerignore ├── .editorconfig ├── .github/ │ ├── copilot-instructions.md │ ├── dependabot.yml │ ├── instructions/ │ │ ├── migrations.instructions.md │ │ ├── pypgstac.instructions.md │ │ ├── scripts.instructions.md │ │ └── sql-source.instructions.md │ ├── prompts/ │ │ ├── add-sql-function.prompt.md │ │ ├── debug-loader.prompt.md │ │ └── stage-version.prompt.md │ └── workflows/ │ ├── continuous-integration.yml │ ├── deploy_mkdocs.yml │ └── release.yml ├── .gitignore ├── .pre-commit-config.yaml ├── AGENTS.md ├── CHANGELOG.md ├── CLAUDE.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── docker/ │ ├── pgstac/ │ │ ├── Dockerfile │ │ └── dbinit/ │ │ └── pgstac.sh │ └── pypgstac/ │ └── Dockerfile ├── docker-compose.yml ├── docs/ │ ├── mkdocs.yml │ └── src/ │ ├── benchmark.json │ ├── item_size_analysis.ipynb │ ├── pgstac.md │ └── pypgstac.md ├── scripts/ │ ├── cibuild │ ├── cipublish │ ├── console │ ├── container-scripts/ │ │ ├── format │ │ ├── initpgstac │ │ ├── loadsampledata │ │ ├── makemigration │ │ ├── pgstac_restore │ │ ├── resetpgstac │ │ ├── stageversion │ │ └── test │ ├── format │ ├── makemigration │ ├── migrate │ ├── pgstacenv │ ├── runinpypgstac │ ├── server │ ├── setup │ ├── stageversion │ ├── test │ └── update └── src/ ├── pgstac/ │ ├── migrations/ │ │ ├── pgstac.0.1.9-0.2.3.sql │ │ ├── pgstac.0.1.9.sql │ │ ├── pgstac.0.2.3-0.2.4.sql │ │ ├── pgstac.0.2.3.sql │ │ ├── pgstac.0.2.4-0.2.5.sql │ │ ├── pgstac.0.2.4-0.2.7.sql │ │ ├── pgstac.0.2.4.sql │ │ ├── pgstac.0.2.5-0.2.7.sql │ │ ├── pgstac.0.2.5.sql │ │ ├── pgstac.0.2.7-0.2.8.sql │ │ ├── pgstac.0.2.7.sql │ │ ├── pgstac.0.2.8-0.2.9.sql │ │ ├── pgstac.0.2.8.sql │ │ ├── pgstac.0.2.9-0.3.0.sql │ │ ├── pgstac.0.2.9.sql │ │ ├── pgstac.0.3.0-0.3.1.sql │ │ ├── pgstac.0.3.0.sql │ │ ├── pgstac.0.3.1-0.3.2.sql │ │ ├── pgstac.0.3.1.sql │ │ ├── pgstac.0.3.2-0.3.3.sql │ │ ├── pgstac.0.3.2.sql │ │ ├── pgstac.0.3.3-0.3.4.sql │ │ ├── pgstac.0.3.3.sql │ │ ├── pgstac.0.3.4-0.3.5.sql │ │ ├── pgstac.0.3.4.sql │ │ ├── pgstac.0.3.5-0.3.6.sql │ │ ├── pgstac.0.3.5.sql │ │ ├── pgstac.0.3.6-0.4.0.sql │ │ ├── pgstac.0.3.6.sql │ │ ├── pgstac.0.4.0-0.4.1.sql │ │ ├── pgstac.0.4.0.sql │ │ ├── pgstac.0.4.1-0.4.2.sql │ │ ├── pgstac.0.4.1.sql │ │ ├── pgstac.0.4.2-0.4.3.sql │ │ ├── pgstac.0.4.2.sql │ │ ├── pgstac.0.4.3-0.4.4.sql │ │ ├── pgstac.0.4.3.sql │ │ ├── pgstac.0.4.4-0.4.5.sql │ │ ├── pgstac.0.4.4.sql │ │ ├── pgstac.0.4.5-0.5.0.sql │ │ ├── pgstac.0.4.5.sql │ │ ├── pgstac.0.5.0-0.5.1.sql │ │ ├── pgstac.0.5.0.sql │ │ ├── pgstac.0.5.1-0.6.0.sql │ │ ├── pgstac.0.5.1.sql │ │ ├── pgstac.0.6.0-0.6.1.sql │ │ ├── pgstac.0.6.0.sql │ │ ├── pgstac.0.6.1-0.6.2.sql │ │ ├── pgstac.0.6.1.sql │ │ ├── pgstac.0.6.10-0.6.11.sql │ │ ├── pgstac.0.6.10.sql │ │ ├── pgstac.0.6.11-0.6.12.sql │ │ ├── pgstac.0.6.11.sql │ │ ├── pgstac.0.6.12-0.6.13.sql │ │ ├── pgstac.0.6.12.sql │ │ ├── pgstac.0.6.13-0.7.0.sql │ │ ├── pgstac.0.6.13-0.7.3.sql │ │ ├── pgstac.0.6.13.sql │ │ ├── pgstac.0.6.2-0.6.3.sql │ │ ├── pgstac.0.6.2.sql │ │ ├── pgstac.0.6.3-0.6.4.sql │ │ ├── pgstac.0.6.3.sql │ │ ├── pgstac.0.6.4-0.6.5.sql │ │ ├── pgstac.0.6.4.sql │ │ ├── pgstac.0.6.5-0.6.6.sql │ │ ├── pgstac.0.6.5.sql │ │ ├── pgstac.0.6.6-0.6.7.sql │ │ ├── pgstac.0.6.6.sql │ │ ├── pgstac.0.6.7-0.6.8.sql │ │ ├── pgstac.0.6.7.sql │ │ ├── pgstac.0.6.8-0.6.9.sql │ │ ├── pgstac.0.6.8.sql │ │ ├── pgstac.0.6.9-0.6.10.sql │ │ ├── pgstac.0.6.9.sql │ │ ├── pgstac.0.7.0-0.7.1.sql │ │ ├── pgstac.0.7.0.sql │ │ ├── pgstac.0.7.1-0.7.2.sql │ │ ├── pgstac.0.7.1.sql │ │ ├── pgstac.0.7.10-0.8.0.sql │ │ ├── pgstac.0.7.10.sql │ │ ├── pgstac.0.7.2-0.7.3.sql │ │ ├── pgstac.0.7.2.sql │ │ ├── pgstac.0.7.3-0.7.4.sql │ │ ├── pgstac.0.7.3.sql │ │ ├── pgstac.0.7.4-0.7.5.sql │ │ ├── pgstac.0.7.4.sql │ │ ├── pgstac.0.7.5-0.7.6.sql │ │ ├── pgstac.0.7.5.sql │ │ ├── pgstac.0.7.6-0.7.7.sql │ │ ├── pgstac.0.7.6.sql │ │ ├── pgstac.0.7.7-0.7.8.sql │ │ ├── pgstac.0.7.7.sql │ │ ├── pgstac.0.7.8-0.7.9.sql │ │ ├── pgstac.0.7.8.sql │ │ ├── pgstac.0.7.9-0.7.10.sql │ │ ├── pgstac.0.7.9.sql │ │ ├── pgstac.0.8.0-0.8.1.sql │ │ ├── pgstac.0.8.0.sql │ │ ├── pgstac.0.8.1-0.8.2.sql │ │ ├── pgstac.0.8.1.sql │ │ ├── pgstac.0.8.2-0.8.3.sql │ │ ├── pgstac.0.8.2.sql │ │ ├── pgstac.0.8.3-0.8.4.sql │ │ ├── pgstac.0.8.3.sql │ │ ├── pgstac.0.8.4-0.8.5.sql │ │ ├── pgstac.0.8.4.sql │ │ ├── pgstac.0.8.5-0.9.0.sql │ │ ├── pgstac.0.8.5.sql │ │ ├── pgstac.0.8.6-0.9.0.sql │ │ ├── pgstac.0.8.6-0.9.10.sql │ │ ├── pgstac.0.8.6.sql │ │ ├── pgstac.0.9.0-0.9.1.sql │ │ ├── pgstac.0.9.0.sql │ │ ├── pgstac.0.9.1-0.9.2.sql │ │ ├── pgstac.0.9.1.sql │ │ ├── pgstac.0.9.10-0.9.11.sql │ │ ├── pgstac.0.9.10.sql │ │ ├── pgstac.0.9.11-unreleased.sql │ │ ├── pgstac.0.9.11.sql │ │ ├── pgstac.0.9.2-0.9.3.sql │ │ ├── pgstac.0.9.2.sql │ │ ├── pgstac.0.9.3-0.9.4.sql │ │ ├── pgstac.0.9.3.sql │ │ ├── pgstac.0.9.4-0.9.5.sql │ │ ├── pgstac.0.9.4.sql │ │ ├── pgstac.0.9.5-0.9.6.sql │ │ ├── pgstac.0.9.5.sql │ │ ├── pgstac.0.9.6-0.9.7.sql │ │ ├── pgstac.0.9.6.sql │ │ ├── pgstac.0.9.7-0.9.8.sql │ │ ├── pgstac.0.9.7.sql │ │ ├── pgstac.0.9.8-0.9.9.sql │ │ ├── pgstac.0.9.8.sql │ │ ├── pgstac.0.9.9-0.9.10.sql │ │ ├── pgstac.0.9.9.sql │ │ └── pgstac.unreleased.sql │ ├── pgstac.sql │ ├── sql/ │ │ ├── 000_idempotent_pre.sql │ │ ├── 001_core.sql │ │ ├── 001a_jsonutils.sql │ │ ├── 001s_stacutils.sql │ │ ├── 002_collections.sql │ │ ├── 002a_queryables.sql │ │ ├── 002b_cql.sql │ │ ├── 003a_items.sql │ │ ├── 003b_partitions.sql │ │ ├── 004_search.sql │ │ ├── 004a_collectionsearch.sql │ │ ├── 005_tileutils.sql │ │ ├── 006_tilesearch.sql │ │ ├── 997_maintenance.sql │ │ ├── 998_idempotent_post.sql │ │ └── 999_version.sql │ └── tests/ │ ├── basic/ │ │ ├── collection_searches.sql │ │ ├── collection_searches.sql.out │ │ ├── cql2_searches.sql │ │ ├── cql2_searches.sql.out │ │ ├── cql_searches.sql │ │ ├── cql_searches.sql.out │ │ ├── crud_functions.sql │ │ ├── crud_functions.sql.out │ │ ├── free_text.sql │ │ ├── free_text.sql.out │ │ ├── partitions.sql │ │ ├── partitions.sql.out │ │ ├── search_path.sql │ │ ├── search_path.sql.out │ │ ├── xyz_searches.sql │ │ └── xyz_searches.sql.out │ ├── pgtap/ │ │ ├── 001_core.sql │ │ ├── 001a_jsonutils.sql │ │ ├── 001b_cursorutils.sql │ │ ├── 001s_stacutils.sql │ │ ├── 002_collections.sql │ │ ├── 002a_queryables.sql │ │ ├── 003_items.sql │ │ ├── 004_search.sql │ │ ├── 004a_collectionsearch.sql │ │ ├── 005_tileutils.sql │ │ ├── 006_tilesearch.sql │ │ ├── 9999_readonly.sql │ │ └── 999_version.sql │ ├── pgtap.sql │ └── testdata/ │ ├── collections.json │ ├── collections.ndjson │ ├── items.ndjson │ ├── items.pgcopy │ ├── items_duplicate_ids.ndjson │ ├── items_for_delsert.ndjson │ └── items_private.ndjson └── pypgstac/ ├── README.md ├── examples/ │ ├── load_queryables_example.py │ └── sample_queryables.json ├── pyproject.toml ├── src/ │ └── pypgstac/ │ ├── __init__.py │ ├── db.py │ ├── hydration.py │ ├── load.py │ ├── migrate.py │ ├── py.typed │ ├── pypgstac.py │ └── version.py └── tests/ ├── __init__.py ├── conftest.py ├── data-files/ │ ├── hydration/ │ │ ├── collections/ │ │ │ ├── chloris-biomass.json │ │ │ ├── landsat-c2-l1.json │ │ │ └── sentinel-1-grd.json │ │ ├── dehydrated-items/ │ │ │ └── landsat-c2-l1/ │ │ │ └── LM04_L1GS_001001_19830527_02_T2.json │ │ └── raw-items/ │ │ ├── landsat-c2-l1/ │ │ │ └── LM04_L1GS_001001_19830527_02_T2.json │ │ └── sentinel-1-grd/ │ │ └── S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C.json │ ├── load/ │ │ └── dehydrated.txt │ └── queryables/ │ └── test_queryables.json ├── hydration/ │ ├── __init__.py │ ├── test_base_item.py │ ├── test_dehydrate.py │ ├── test_dehydrate_pg.py │ ├── test_hydrate.py │ └── test_hydrate_pg.py ├── test_benchmark.py ├── test_load.py └── test_queryables.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .devcontainer/devcontainer.json ================================================ { "name": "PgSTAC", "dockerComposeFile": "../docker-compose.yml", "service": "pgstac", "workspaceFolder": "/opt/src" } ================================================ FILE: .dockerignore ================================================ .envrc* */dist/* *.pyc *.egg-info *.eggs venv/* */.direnv/* */.ruff_cache/* */.pytest_cache/* */.vscode/* */.mypy_cache/* */.pgadmin/* */.ipynb_checkpoints/* */.git/* */.github/* */env/* Dockerfile docker compose.yml */.devcontainer/* ================================================ FILE: .editorconfig ================================================ root = true [*] end_of_line = lf insert_final_newline = true charset = utf-8 trim_trailing_whitespace = true [*.sql] indent_style = space indent_size = 4 [*.py] indent_style = space indent_size = 4 [*.{yml,yaml}] indent_style = space indent_size = 2 [*.md] trim_trailing_whitespace = false [Makefile] indent_style = tab [*.sh] indent_style = space indent_size = 4 ================================================ FILE: .github/copilot-instructions.md ================================================ # Copilot Instructions for PgSTAC See `CLAUDE.md` for comprehensive project instructions, architecture, and workflows. See `AGENTS.md` for specialized agent definitions (sql-developer, migration-engineer, loader-developer). ================================================ FILE: .github/dependabot.yml ================================================ version: 2 updates: - package-ecosystem: "github-actions" directory: "/" schedule: interval: "weekly" day: "monday" open-pull-requests-limit: 5 groups: actions-all: applies-to: version-updates patterns: - "*" - package-ecosystem: "docker" directory: "/" schedule: interval: "weekly" day: "monday" open-pull-requests-limit: 5 groups: docker-base-images: applies-to: version-updates patterns: - "*" - package-ecosystem: "pip" directory: "/src/pypgstac" schedule: interval: "weekly" day: "monday" open-pull-requests-limit: 5 groups: python-dev-tooling: applies-to: version-updates patterns: - "ruff" - "ty" - "pre-commit" - "types-*" python-runtime: applies-to: version-updates patterns: - "cachetools" - "fire" - "hydraters" - "orjson" - "plpygis" - "pydantic" - "python-dateutil" - "smart-open" - "tenacity" - "version-parser" - "psycopg*" ================================================ FILE: .github/instructions/migrations.instructions.md ================================================ --- applyTo: "src/pgstac/migrations/**" --- # Migration Files These files are **generated** — see CLAUDE.md "Migration Process" for the full workflow. - **DO NOT** create, edit, or hand-modify migration files - Base (`pgstac.X.Y.Z.sql`) = full schema at that version - Incremental (`pgstac.X.Y.Z-A.B.C.sql`) = upgrade diff - Staged (`*.sql.staged`) = needs review before removing `.staged` suffix - Test: `scripts/test --migrations` ================================================ FILE: .github/instructions/pypgstac.instructions.md ================================================ --- applyTo: "src/pypgstac/**" --- # pypgstac Python See CLAUDE.md "pypgstac Loader Internals" for patterns. See AGENTS.md "loader-developer" for critical rules. - Uses psycopg v3 (not psycopg2), orjson (not json), tenacity, plpygis, fire - Materialize generators before retry boundaries - Query `partition_sys_meta` (live VIEW), never `partitions` (stale MATERIALIZED VIEW) - Test: `scripts/runinpypgstac --build test --pypgstac` ================================================ FILE: .github/instructions/scripts.instructions.md ================================================ --- applyTo: "scripts/**" --- # Build Scripts See CLAUDE.md "Development Workflow" for usage. All scripts require the Docker compose environment. - `runinpypgstac` is the foundation — most scripts delegate to it - `scripts/container-scripts/` contains the in-container script payload copied into the pypgstac image; keep host wrappers in `scripts/` - `stageversion` modifies version files AND generates migrations — see CLAUDE.md "Migration Process" - DO NOT run `stageversion` without understanding its side effects ================================================ FILE: .github/instructions/sql-source.instructions.md ================================================ --- applyTo: "src/pgstac/sql/**" --- # SQL Source Files See CLAUDE.md "Critical Rules" for full SQL conventions. - NEVER edit `pgstac.sql` — it is auto-generated - `CREATE OR REPLACE FUNCTION`, `IF NOT EXISTS`, `SECURITY DEFINER` - Grant permissions in `998_idempotent_post.sql`, not inline - `get_tstz_constraint()` regex must handle fractional seconds (`.` in timestamps) - Do NOT schema-qualify PostGIS calls — PostGIS may be in `public` or `postgis` schema - SQL functions used by GENERATED columns must be self-contained (no cross-function deps) — pg_dump orders functions alphabetically and breaks dependency chains - Test: `scripts/runinpypgstac --build test --pgtap --basicsql` ================================================ FILE: .github/prompts/add-sql-function.prompt.md ================================================ --- description: "Add a new SQL function to PgSTAC" --- Add a new SQL function following PgSTAC conventions: 1. Determine the correct file in `src/pgstac/sql/` based on the prefix ranges in CLAUDE.md 2. Use `CREATE OR REPLACE FUNCTION` in the `pgstac` schema 3. Add `SECURITY DEFINER` if the function modifies tables 4. Add permission grants in `src/pgstac/sql/998_idempotent_post.sql` if the function should be callable by `pgstac_ingest` or `pgstac_read` 5. Add a test in `src/pgstac/tests/pgtap.sql` or a new `.sql`/`.sql.out` pair in `src/pgstac/tests/basic/` Test with: `scripts/runinpypgstac --build test --pgtap --basicsql` ================================================ FILE: .github/prompts/debug-loader.prompt.md ================================================ --- description: "Debug a pypgstac data loading issue" --- Diagnose a pypgstac Loader problem: 1. Check which load mode is being used (`insert`, `ignore`, `upsert`, `delsert`) 2. Verify the loader queries `partition_sys_meta` (live VIEW), not `partitions` (stale MATERIALIZED VIEW) 3. Check that generators are materialized to `list()` before `load_partition()` — generators can't survive tenacity retries 4. Verify `item.pop("partition", None)` uses `None` default for retry safety 5. Check the retry decorator covers: `CheckViolation`, `DeadlockDetected`, `SerializationFailure`, `LockNotAvailable`, `ObjectInUse` 6. Check `before_sleep` handler sets `partition.requires_update = True` on `CheckViolation` 7. Verify `get_tstz_constraint()` regex handles fractional seconds (`.` in timestamps) Key files: - `src/pypgstac/src/pypgstac/load.py` — Loader class - `src/pgstac/sql/003b_partitions.sql` — partition constraint functions ================================================ FILE: .github/prompts/stage-version.prompt.md ================================================ --- description: "Stage a new PgSTAC version and review the migration" --- Guide me through the PgSTAC release migration process: 1. Confirm all SQL changes are in `src/pgstac/sql/` (never `pgstac.sql` directly) 2. Run `scripts/stageversion {VERSION}` to generate base + incremental migrations 3. Review the `.staged` migration file checking for: - Unintended `DROP TABLE` or `DROP COLUMN` - Unsafe `ALTER TABLE` for large tables - Bare `CREATE` instead of `CREATE OR REPLACE` for functions - Missing `IF NOT EXISTS` on indexes - Presence of `000_idempotent_pre.sql` and `998_idempotent_post.sql` content - `set_version()` called at the end 4. Remove the `.staged` suffix 5. Run `scripts/test --migrations` to validate the full migration chain 6. Update CHANGELOG.md ================================================ FILE: .github/workflows/continuous-integration.yml ================================================ name: CI on: push: branches: - main pull_request: workflow_dispatch: schedule: - cron: '23 4 * * 0' env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} DOCKER_BUILDKIT: 1 PIP_BREAK_SYSTEM_PACKAGES: 1 jobs: changes: runs-on: ubuntu-latest permissions: pull-requests: read outputs: pgtagprefix: ${{ steps.check.outputs.pgtagprefix }} buildpgdocker: ${{ steps.check.outputs.buildpg }} pyrustdocker: ${{ steps.check.outputs.pytag }} buildpyrustdocker: ${{ steps.check.outputs.buildpy }} steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: dorny/paths-filter@fbd0ab8f3e69293af611ebaee6363fc25e6d187d # v4.0.1 id: filter with: filters: | pgstac: - 'docker/pgstac/**' pypgstac: - 'docker/pypgstac/**' - id: check run: | buildpg=false; ref=$(echo ${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} | tr / _); pgref=$ref; [[ "${{ steps.filter.outputs.pgstac }}" == "true" ]] && buildpg=true || pgref=main; if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then buildpg=true; pgref=main; fi echo "pgtagprefix=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-postgres:$pgref" >>$GITHUB_OUTPUT; echo "buildpg=$buildpg" >>$GITHUB_OUTPUT; buildpy=false; pyref=$ref; [[ "${{ steps.filter.outputs.pypgstac }}" == "true" ]] && buildpy=true || pyref=main; if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then buildpy=true; pyref=main; fi echo "pytag=${{ env.REGISTRY }}/${GITHUB_REPOSITORY_OWNER}/pgstac-pyrust:$pyref" >>$GITHUB_OUTPUT; echo "buildpy=$buildpy" >>$GITHUB_OUTPUT; # This builds a base postgres image that has everything installed to be able to run pgstac. This image does not have pgstac itself installed. buildpg: name: Build and push base postgres image runs-on: ubuntu-latest needs: [changes] strategy: matrix: pg_major: [16, 17, 18] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to the Container registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and Push Base Postgres if: ${{ needs.changes.outputs.buildpgdocker == 'true' }} uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: platforms: linux/amd64,linux/arm64 context: . target: pgstacbase file: docker/pgstac/Dockerfile build-args: | PG_MAJOR=${{ matrix.pg_major }} tags: ${{ needs.changes.outputs.pgtagprefix }}-pg${{ matrix.pg_major }} push: true cache-from: type=gha cache-to: type=gha, mode=max buildpyrust: name: Build and push base pyrust runs-on: ubuntu-latest needs: [changes] steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to the Container registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and Push Base pyrust if: ${{ needs.changes.outputs.buildpyrustdocker == 'true' }} uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: platforms: linux/amd64,linux/arm64 context: . target: pyrustbase file: docker/pypgstac/Dockerfile tags: ${{ needs.changes.outputs.pyrustdocker }} push: true cache-from: type=gha cache-to: type=gha, mode=max test: name: test needs: [changes, buildpg, buildpyrust] runs-on: ubuntu-latest strategy: matrix: pg_major: [16, 17, 18] flags: - "" - "--resolution lowest-direct" container: image: ${{ needs.changes.outputs.pyrustdocker }} options: --user root env: PGPASSWORD: postgres PGHOST: postgres PGDATABASE: postgres PGUSER: postgres UV_CACHE_DIR: /tmp/.uv-cache services: postgres: env: POSTGRES_PASSWORD: postgres image: ${{ needs.changes.outputs.pgtagprefix }}-pg${{ matrix.pg_major }} options: >- --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Ensure PostgreSQL client tools run: | set -euo pipefail apt-get update apt-get install -y --no-install-recommends gnupg ca-certificates curl install -d -m 0755 /etc/apt/keyrings curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc | gpg --dearmor -o /etc/apt/keyrings/postgresql.gpg . /etc/os-release echo "deb [signed-by=/etc/apt/keyrings/postgresql.gpg] http://apt.postgresql.org/pub/repos/apt ${VERSION_CODENAME}-pgdg main" > /etc/apt/sources.list.d/pgdg.list apt-get update apt-get install -y --no-install-recommends "postgresql-client-${{ matrix.pg_major }}" client_major="$(pg_dump --version | sed -E 's/.* ([0-9]+)\..*/\1/')" server_major="$(psql -X -tA -h postgres -d postgres -c 'show server_version' | sed -E 's/^([0-9]+).*/\1/')" if [[ "$client_major" != "${{ matrix.pg_major }}" ]]; then echo "Expected pg_dump major ${{ matrix.pg_major }}, got ${client_major}" >&2 exit 1 fi if [[ "$client_major" != "$server_major" ]]; then echo "pg_dump major (${client_major}) does not match postgres major (${server_major})" >&2 exit 1 fi - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0 - name: Install pypgstac working-directory: /__w/pgstac/pgstac/src/pypgstac run: | export UV_CACHE_DIR=/tmp/.uv-cache export XDG_CACHE_HOME=/tmp/.cache mkdir -p "$UV_CACHE_DIR" "$XDG_CACHE_HOME" uv --cache-dir "$UV_CACHE_DIR" venv /tmp/ci-venv uv --cache-dir "$UV_CACHE_DIR" pip install --python /tmp/ci-venv/bin/python ${{ matrix.flags }} .[dev,test,psycopg] echo "/tmp/ci-venv/bin" >> "$GITHUB_PATH" - name: Run tests working-directory: /__w/pgstac/pgstac run: scripts/container-scripts/test ================================================ FILE: .github/workflows/deploy_mkdocs.yml ================================================ name: Publish docs via GitHub Pages on: push: branches: - main paths: - 'README.md' - 'CHANGELOG.md' - 'CONTRIBUTING.md' - 'docs/**' pull_request: paths: - 'README.md' - 'CHANGELOG.md' - 'CONTRIBUTING.md' - 'docs/**' jobs: docs: name: ${{ github.event_name == 'push' && 'Deploy docs' || 'Build docs' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Set up Python 3.12 uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: 3.12 - name: Install dependencies run: | python -m pip install --upgrade pip python -m pip install mkdocs mkdocs-material mkdocs-jupyter pandas seaborn folium - name: Build docs if: github.event_name == 'pull_request' run: mkdocs build -f docs/mkdocs.yml - name: Deploy docs if: github.event_name == 'push' run: mkdocs gh-deploy --force -f docs/mkdocs.yml ================================================ FILE: .github/workflows/release.yml ================================================ name: Release on: push: tags: - "*" env: REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} jobs: tag: name: tag runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Tag Release uses: "marvinpinto/action-automatic-releases@919008cf3f741b179569b7a6fb4d8860689ab7f0" # v1.2.1 with: repo_token: "${{ secrets.GITHUB_TOKEN }}" prerelease: false # This builds a base postgres image that has everything installed to be able to run pgstac. buildpg: name: Build and push base postgres image runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 # Temporary workaround for ARM builds # https://github.com/docker/setup-qemu-action/issues/198 with: image: tonistiigi/binfmt:qemu-v7.0.0-28 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to the Container registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-postgres - name: Build and Push Base Postgres uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: platforms: linux/amd64,linux/arm64 context: . target: pgstacbase file: docker/pgstac/Dockerfile tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} push: true cache-from: type=gha cache-to: type=gha, mode=max # This builds a postgres image that already has pgstac installed to the tagged version buildpgstac: name: Build and push base postgres with pgstac installed runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 # Temporary workaround for ARM builds # https://github.com/docker/setup-qemu-action/issues/198 with: image: tonistiigi/binfmt:qemu-v7.0.0-28 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to the Container registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - name: Build and Push Base Postgres uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: platforms: linux/amd64,linux/arm64 context: . target: pgstac file: docker/pgstac/Dockerfile tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} push: true cache-from: type=gha cache-to: type=gha, mode=max buildpyrust: name: Build and push base pyrust runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 # Temporary workaround for ARM builds # https://github.com/docker/setup-qemu-action/issues/198 with: image: tonistiigi/binfmt:qemu-v7.0.0-28 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to the Container registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-pyrust - name: Build and Push Base Postgres uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: platforms: linux/amd64,linux/arm64 context: . target: pyrustbase file: docker/pypgstac/Dockerfile tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} push: true cache-from: type=gha cache-to: type=gha, mode=max buildpypgstac: name: Build and push base pyrust runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 # Temporary workaround for ARM builds # https://github.com/docker/setup-qemu-action/issues/198 with: image: tonistiigi/binfmt:qemu-v7.0.0-28 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to the Container registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-pypgstac - name: Build and Push Base Postgres uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: platforms: linux/amd64,linux/arm64 context: . target: pypgstac file: docker/pypgstac/Dockerfile tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} push: true cache-from: type=gha cache-to: type=gha, mode=max buildpypgstacruntime: name: Build and push pypgstac runtime image runs-on: ubuntu-latest steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - uses: docker/setup-qemu-action@ce360397dd3f832beb865e1373c09c0e9f86d70a # v4.0.0 with: image: tonistiigi/binfmt:qemu-v7.0.0-28 - uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0 - name: Log in to the Container registry uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0 with: registry: ${{ env.REGISTRY }} username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata (tags, labels) for Docker id: meta uses: docker/metadata-action@030e881283bb7a6894de51c315a6bfe6a94e05cf # v6.0.0 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}-pypgstac-runtime - name: Build and Push pypgstac runtime uses: docker/build-push-action@bcafcacb16a39f128d818304e6c9c0c18556b85f # v7.1.0 with: platforms: linux/amd64,linux/arm64 context: . target: pypgstac-runtime file: docker/pypgstac/Dockerfile tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} push: true cache-from: type=gha cache-to: type=gha, mode=max releasetopypi: name: Release runs-on: ubuntu-latest permissions: id-token: write environment: name: pypi url: https://pypi.org/p/pypgstac steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: Setup Python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 with: python-version: "3.x" - name: Install build working-directory: /home/runner/work/pgstac/pgstac/src/pypgstac run: pip install build - name: Build working-directory: /home/runner/work/pgstac/pgstac/src/pypgstac run: python -m build - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # release/v1 with: packages-dir: /home/runner/work/pgstac/pgstac/src/pypgstac/dist ================================================ FILE: .gitignore ================================================ .so .envrc src/pypgstac/dist *.pyc *.egg-info *.eggs venv env .direnv src/pypgstac/target src/pypgstac/python/pypgstac/*.so .vscode .ipynb_checkpoints .venv .pytest_cache .plans/ .env src/pgstacrust/target/ ================================================ FILE: .pre-commit-config.yaml ================================================ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v5.0.0 hooks: - id: trailing-whitespace - id: check-yaml - id: check-added-large-files - id: check-toml - id: detect-aws-credentials args: [--allow-missing-credential] - id: detect-private-key - id: check-json - id: mixed-line-ending - id: check-merge-conflict - id: check-executables-have-shebangs - id: check-symlinks - repo: local hooks: - id: dockerbuild name: dockerbuild entry: scripts/update language: script pass_filenames: false verbose: true fail_fast: true files: Dockerfile$|\.rs$ - id: sql name: sql entry: scripts/test args: [--basicsql, --pgtap] language: script pass_filenames: false verbose: true fail_fast: true files: sql\/.*\.sql$ - id: formatting name: formatting entry: scripts/test args: [--formatting] language: script pass_filenames: false verbose: true fail_fast: true files: ^(src/pypgstac/(src/pypgstac|tests)/.*\.py|src/pypgstac/pyproject\.toml)$ - id: pypgstac name: pypgstac entry: scripts/test args: [--pypgstac] language: script pass_filenames: false verbose: true fail_fast: true files: pypgstac\/.*\.py$ - id: migrations name: migrations entry: scripts/test args: [--migrations] language: script pass_filenames: false verbose: true fail_fast: true files: migrations\/.*\.sql$ ================================================ FILE: AGENTS.md ================================================ # PgSTAC Agents ## sql-developer PostgreSQL SQL developer for PgSTAC. Works exclusively in `src/pgstac/sql/` files. See CLAUDE.md for full SQL rules, file map, and partition architecture. ### Key Constraints - NEVER edit `pgstac.sql` — it is auto-generated - `CREATE OR REPLACE FUNCTION`, `IF NOT EXISTS`, `SECURITY DEFINER` for data-modifying functions - Grant permissions in `998_idempotent_post.sql`, not inline - Use `run_or_queue()` for deferrable operations - Do NOT schema-qualify PostGIS calls (PostGIS may be in `public` or `postgis` schema) - Avoid cross-function deps in SQL functions used by GENERATED columns — pg_dump orders alphabetically, so inline the logic (see `search_hash` pattern) - Test: `scripts/runinpypgstac --build test --pgtap --basicsql` --- ## migration-engineer Migration specialist for PgSTAC. See CLAUDE.md "Migration Process" for full workflow. ### Quick Reference 1. Edit SQL in `src/pgstac/sql/*.sql` 2. `scripts/stageversion VERSION` → generates base + incremental `.staged` migration 3. Review `.staged` file (watch for DROPs, unsafe ALTERs, missing `CREATE OR REPLACE`) 4. Remove `.staged` suffix → `scripts/test --migrations` ### Review Checklist - No unintended `DROP TABLE/COLUMN`, safe `ALTER TABLE` for large tables - `CREATE OR REPLACE` (not bare `CREATE`), `IF NOT EXISTS` for indexes - `000_idempotent_pre.sql` and `998_idempotent_post.sql` included - `set_version()` called at end --- ## loader-developer Specialist in pypgstac bulk loading (`src/pypgstac/src/pypgstac/load.py`). See CLAUDE.md "pypgstac Loader Internals" for full details. ### Critical Patterns - **Materialize generators**: `list(g)` before `load_partition()` — generators can't survive tenacity retries - **Live view only**: Query `partition_sys_meta` (VIEW), never `partitions` (stale MATERIALIZED VIEW) - **Retry safety**: `item.pop("partition", None)` with `None` default; `before_sleep` sets `partition.requires_update = True` on `CheckViolation` - **Retry scope**: `CheckViolation`, `DeadlockDetected`, `SerializationFailure`, `LockNotAvailable`, `ObjectInUse` - **Load modes**: `insert`, `ignore`/`insert_ignore`, `upsert`, `delsert` - Test: `scripts/runinpypgstac --build test --pypgstac` ================================================ FILE: CHANGELOG.md ================================================ # Changelog All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). ## [Unreleased] ### Added - `scripts/makemigration` host wrapper for the in-container `makemigration` helper. - `.env.example` documenting all supported environment variables for local development. - All host-facing scripts (`test`, `format`, `migrate`, `server`, `stageversion`, `runinpypgstac`, `console`) now accept `--help` / `-h` and honor environment-variable counterparts for common flags (`PGSTAC_BUILD_POLICY`, `PGSTAC_FAST`, `PGSTAC_WATCH`, `PGSTAC_STRICT`). - `scripts/pgstacenv` gains `ensure_env_file` (auto-creates `.env` from `.env.example` on first run) and `first_available_pgport` (avoids port collisions on shared machines). - `scripts/test` expanded from a 3-line wrapper to a full-featured test runner supporting `--fast`, `--watch`, `--build-policy`, `--no-strict`, and stale-image detection. - PostgreSQL 16, 17, and 18-beta added to the CI and Docker build matrix. (Closes #334) - Weekly scheduled CI run (`cron: '23 4 * * 0'`) to catch upstream base-image CVEs without requiring a code change. (Closes #202) - `workflow_dispatch` trigger for manual CI runs. - `pg_tle` v1.5.2 built and pre-loaded in the `pgstacbase` image; database init runs `CREATE EXTENSION IF NOT EXISTS pg_tle`. - `pypgstac-runtime` Docker target: slim Python 3.13-trixie image without the Rust/build toolchain, for production deployments where the Rust build environment is not needed. - Dependabot coverage expanded to Docker base images and pip packages (two new ecosystems with grouped update policies). ### Changed - In-container helper scripts moved from `docker/pypgstac/bin/` to `scripts/container-scripts/`; container `PATH` updated accordingly. - `docker/pgstac/Dockerfile` and `docker/pypgstac/Dockerfile` base images updated from `bullseye` to `trixie`. (Closes #231) - All Docker `RUN` layers now use BuildKit cache mounts for apt, uv, and git caches, significantly reducing incremental rebuild times. - `docker-compose.yml`: adds `env_file: .env`, explicit `PGHOST`/`PGPORT` defaults, a pgstac healthcheck, and a `service_healthy` dependency on pypgstac. - `runinpypgstac` gains `--build-policy {always,missing,never}` replacing the bare `--build` flag; `PGSTAC_BUILD_POLICY` env var provides a persistent default. - Dev tooling: `flake8`, `black`, and `mypy` removed in favour of `ruff==0.15.11` and `ty==0.0.31`. `pre-commit` pinned to `3.5.0`. `pre-commit-hooks` updated to v5.0.0. - `pypgstac` package floor raised to Python 3.11; metadata now advertises 3.11-3.14. - `pypgstac` settings now use `pydantic-settings` (`BaseSettings` from `pydantic_settings`) and require `pydantic>=2,<3`. - `cachetools` upper bound removed (`cachetools>=5.3.0`) since `pypgstac` only uses `cachetools.func.lru_cache`; no known incompatible API changes affect this usage. - `pypgstac` developer tooling config now consistently targets Ruff + ty: removes stale mypy config, pins Ruff to `0.15.11` to match pre-commit, and adds minimal `[tool.ty]` project settings. - Formatting/type-check pipeline now uses `scripts/test --formatting` as the single pre-commit entry point (removing duplicate direct Ruff pre-commit hooks) and aligns Ruff line-length handling with the formatter (`E501` ignored; explicit `line-length = 88`). - GitHub Actions updated: `dorny/paths-filter` v2→v3, `docker/build-push-action` v4→v6, `astral-sh/setup-uv` v8.0.0→v8.1.0; all SHA pins refreshed. - Dependabot groups reworked: `actions-all` (replaces `minor-and-patch`), new `docker-base-images`, `python-dev-tooling`, and `python-runtime` groups. - `docker-compose.yml` removes explicit `container_name` entries to avoid conflicts between concurrent local instances. ### Removed - PL/Rust support: `pgstacbase-plrust` and `pgstac-plrust` Docker targets removed; the pgstac image no longer builds or ships PL/Rust or the Rust toolchain. (Closes #339) - `flake8`, `black`, and `mypy` removed from dev dependencies. ### Fixed - `load.py`: Use timezone-aware `MIN_DATETIME_UTC` / `MAX_DATETIME_UTC` sentinel constants (instead of naive `datetime.min` / `datetime.max`) to avoid `TypeError: can't compare offset-naive and offset-aware datetimes`. - CI `lowest-direct` dependency install: avoid uv cache permission failures by using a writable temp cache path in the test job install step. - `pypgstac` dependency floor for `orjson` raised to `>=3.11.0` to avoid selecting the broken `3.9.0` sdist under `--resolution lowest-direct`. - `pydantic` minimum raised to `>=2.10` so `--resolution lowest-direct` on Python 3.13 does not resolve to `pydantic-core==2.0.1`, which fails to build. ## [v0.9.11] ### Fixed - Fix timestamp regex in partition constraint parsing to handle fractional seconds (microseconds), preventing incorrect `(-infinity, infinity)` constraint bounds. - Add explicit ANALYZE before `st_estimatedextent()` in `update_partition_stats` for deterministic spatial extent calculation. - Consolidate materialized view refreshes in `update_partition_stats` to a single unconditional refresh, reducing redundant operations. - Use `partition_sys_meta` (live VIEW) instead of `partitions` (stale MATERIALIZED VIEW) in loader `_partition_update()` for real-time partition bounds. - Expand loader retry to 10 attempts and add `SerializationFailure`, `LockNotAvailable`, `ObjectInUse` to retryable exceptions. - Add `before_sleep` retry handler to force partition constraint refresh on `CheckViolation`. - Materialize `itertools.groupby` generators with `list()` before `load_partition()` to prevent silent data loss on retry. - Use safe `item.pop('partition', None)` to avoid `KeyError` on retry. - Inline `search_tohash` into `search_hash` to eliminate cross-function dependency that broke pg_dump/pg_restore (pg_dump orders functions alphabetically). ### Added - `pgstac_restore` script for restoring pg_dump backups — installs a temporary event trigger to fix search_path during restore. - Race condition tests for sequential and concurrent loader operations with new-Loader-per-item pattern. - Search path independence tests verifying `partition_sys_meta`, `partition_stats`, `partitions`, `partitions_view`, and `partition_steps` work identically with and without `pgstac` in `search_path`. - pg_dump/pg_restore test (`--pgdump`) validating backup and restore of a pgstac database with sample data. - Documentation for pg_dump/pg_restore best practices with PgSTAC. ## [v0.9.10] ### Fixed - Improved performance and correctness of partition constraint parsing. ##[v0.9.9] ### Changed * changed container images to use non-root `user` * fix bug where closed and broken db connections are reused. ### Fixed * replace space-separated terms with adjacency operator in free-text search (#387) ## [v0.9.8] ### Fixed - Allow array as q parameter for full text search ## [v0.9.7] ### Fixed - Fix bad handling of leading +/- terms in free-text search - Use consistent tsquery config in free-text search ## [v0.9.6] ### Added - Add `load_queryables` function to pypgstac for loading queryables from a JSON file - Add support for specifying collection IDs when loading queryables ### Fixed - Added missing 0.8.6-0.9.0 migration script ## [v0.9.5] ### Changed - Pin to `plpygic>=0.5.0` and use `geom.ewkb` instead of `geom.wkt` when formatting items in `Loader.format_item`. Fixes (#357) ## [v0.9.4] ### Changed - Relax pypgstac dependencies ## [v0.9.3] ### Fixed - Fix CI issue with tests not running - Fix for issue with nulls in title or keywords for free text search ### Changed - Replace hardcoded org name in CI ## [v0.9.2] ### Added - Add limited support for free-text search in the search functions. (Fixes #293) - the `q` parameter is converted from the [OGC API - Features syntax](https://docs.ogc.org/DRAFTS/24-031.html) into a `tsquery` statement which is used to compare to the description, title, and keywords fields in items or collection_search - the text search is un-indexed and will be very slow for item-level searches! - Add support for Postgres 17 - Support for adding data to the private field using the pypgstac loader ### Fixed - Add `open=True` in `psycopg.ConnectionPool` to avoid future behavior change - Switch from postgres `server_version` to `server_version_num` to get PG version (Fixes #300) - Allow read-only replicas work even when the context extension is enabled (Fixes #300) - Consistently ensure use of instantiated postgres fields when addressing with 'properties.' prefix ### Changed - Move rust hydration to a separate repo ## [v0.9.1] ### Fixed - Fixed double nested extent when using trigger based update collection extent. (Fixes #274) - Fix time formatting (Fixes #275) - Relaxes smart-open dependency check (Fixes #273) - Switch to uv for docker image ## [v0.9.0] ### Breaking Changes - Context Extension has been deprecated. Context is now reported using OGC Features compliant numberMatched and numberReturned - Paging return from search using prev/next properties has been deprecated. Paging is now available in the spec compliant Links ### Added - Add support for Casei and Accenti (Fixes #237). (Also, requires the addition of the unaccent extension) - Add numberReturned and numberMatched fields for ItemCollection. BREAKING CHANGE: As the context extension is deprecated, this also removes the "context" item from results. - Updated docs on automated updates of collection extents. (CLOSES #247) - stac search now returns paging information using standards compliant links rather than prev/next properties (Fixes #265) ### Fixed - Fixes issue when there is a None rather than an empty dictionary in hydration. - Use "debug" log level rather than "log" to prevent growth in log messages due to differences in how client_min_messages and log_min_messages treat log levels. (Fixes #242) - Refactor search_query and search_where functions to eliminate race condition when running identical queries. (Fixes #233) - Fixes CQL2 Parser for Between operator (Fixes #251) - Update PyO3 for rust hydration performance improvements. ## [v0.8.6] ### Fixed - Relax version requirement for smart-open (Fixes #273) - Use uv pip in docker build ## [v0.8.5] ### Fixed - Fix issue when installing or migrating pgstac using a non superuser (particularly when using the default role found on RDS). (FIXES #239). Backports fix into migrations for 0.8.2, 0.8.3, and 0.8.4. - Adds fixes/updates to documentation - Fixes issue when using geometry with the strict queryables setting set. ## [v0.8.4] ### Fixed - Make release deployment use postgres images without plrust - Update versions of plrust in dockerfile (used for development, there is no plrust code yet) - Update incremental migration tests to start at v0.3.0 rather than v0.1.9 due to a breaking change in pg_partman at version 5 that has no ability to pin a version. Migrating from prior to v0.3.0 should still work fine as long as pg_partman has not been updated on the database. ## [v0.8.3] ### Added - Add support for arm64 to Docker images ### Fixed - Fixes a critical bug when using the ingest_staging_upsert table or the upsert_item/upsert_items functions to update records with existing data where the existing row would get deleted, but the new row would not get added. ## [v0.8.2] ### Added - Add support functions and tests for Collection Search - Add configuration parameter for base_url to be able to generate absolute links - With this release, this is only used to create links for paging in collection_search - Adds read only mode to allow use of pgstac on read replicas - Note: Turning on romode disables any caching (particularly when context is turned on) and does not allow to store q query hash that can be used with geometry_search. - Add option to pypgstac loader "--usequeue" that forces use of the query queue for the loading process - Add "pypgstac runqueue" command to run any commands that are set in the query queue ### Fixed - Fix bug with end_datetime constraint management leading to inability to add data outside of constraints - Fix bugs dealing with table ownership to ensure that all pgstac tables are owned by the pgstac_admin role - Fixes issues with errors/warnings caused when doing index maintenance - Fixes issues with errors/warnings caused with partition management - Make sure that pgstac_ingest role always has read/write permissions on all tables - Remove call to create_table_constraints from check_partition function. create_table_constraints was being called twice as it also gets called from update_partition_stats - Add NOT NULL constraint to collections table (FIXES #224) - Fix issue with indexes not getting created as the pg_admin role using SECURITY DEFINER ### Changed - Revert pydantic requirement back to '>=1.7' and use basesettings conditionally from pydantic or pydantic.v1 to allow compatibility with pydantic 2 as well as with stac-fastapi that requires pydantic <2 ## [v0.8.1] ### Fixed - Fix issue with CI building/pushing docker images ## [v0.8.0] ### Fixed - Revert an optimisation which limited the number of results from a search query to the number of item IDs specified in the query. This fixes an issue where items with the same ID that are in multiple collections could be left out of search results. ### Changed - update `pydantic` requirement to `~=2.0` - update docker and ci workflows to build binary wheels for rust additions to pypgstac - split docker into database service and python/rust container - Modify scripts to auto-generate unreleased migration - Add pre commit tasks to generate migration and to rebuild and compile pypgstac with maturin for rust - Add private jsonb column to items and collections table to hold private metadata that should not be returned as part of a stac item - Add generated columns to collections with the bounding box as a geometry and the datetime and end_datetime from the extents (this is to help with forthcoming work on collections search) - Add PLRust to the Docker postgres image for forthcoming work to add optional PLRust functions for expensive json manipulation (including hydration) - Remove default queryable for eo:cloud_cover ## [v0.7.10] ### Fixed - Return an empty jsonb array from all_collections() when the collections table is empty, instead of NULL. Fixes #186. - Add delete trigger to collections to clean up partition_stats records and remove any partitions. Fixes #185 - Fixes boolean casting in get_setting_bool function ## [v0.7.9] ### Fixed - Update docker image to use postgis 3.3.3 ## [v0.7.8] ### Fixed - Fix issue with search_query not returning all fields on first use of a query. Fixes #182 ## [v0.7.7] ### Fixed - Fix migrations for 0.7.4->0.7.5 and 0.7.5->0.7.6 to use the partition_view rather than the materialized view to avoid issue with refreshing the materialized view when run in the same statement that is accessing the view. Fixes #180. ### Added - Add a short cirucit for id searches that sets the limit to be no more than the number of ids in the filter. - Add 'timing' configuration variable that adds a "timing" element to the return object with the amount of time that it took to return a search. - Reduce locking when updating statistics in the search table. Use skip locked to skip updating last_used and count when there is a lock being held. ## [v0.7.6] ### Fixed - Fix issue with checking for existing collections in queryable trigger function that prevented adding scoped queryable entries. ## [v0.7.5] ### Fixed - Default sort not getting set when sortby not included in query with token (Fixes [#177](https://github.com/stac-utils/pgstac/issues/177)) - Fixes regression in performance between with changes for partition structure at v0.7.0. Changes the normal view for partitions and partition_steps into indexed materialized views. Adds refreshing of the views to existing triggers to make sure they stay up to date. ## [v0.7.4] ### Added - Add --v and --vv options to scripts/test to change logging to notice / log when running tests. - Add framework for option to cache expensive item formatting/hydrating calls. Note: this only provides functionality to add and read from the cached calls, but does not have any wiring to remove any entries from the cache. - Update the costs for json formatting functions to 5000 to help the query planner choose to prefer using indexes on json fields. ### Fixed - Fix bug in foreign key and unique collection detection in queryables trigger function, update tests to catch. - Add collection id to tokens to ensure uniqueness and improve speed when looking up token values. Update tests to use the new keys. Old item id only tokens are still valid, but new results will all contain the new keys. - Improve performance when looking for whether next/prev links should be added. - Update Search function to remove the use of cursors and temp tables. - Update get_token_filter to remove the use of temp tables. ## [v0.7.3] ### Fixed - Use IF EXISTS when dropping constraints to avoid race conditions - Rework function that finds indexes that need to be added to be added and to find functionally identical indexes better. ## [v0.7.2] ### Fixed - Use version_parser for parsing versions in pypgstac - Fix issue with dropping functions/procedures in 0.6.13->0.7.0 migrations - Fix issue with CREATE OR REPLACE TRIGGER on PG 13 - Fix issue identifying duplicate indexes in maintain_partition_queries function - Ensure that pgstac_read role has read permissions to all partitions - Fix issue (and add tests) caused by bug in psycopg datetime types not being able to translate 'infinity', '-infinity' ## [v0.7.1] ### Fixed - Fix permission issue when running incremental migrations. - Make sure that pypgstac migrate runs in a single transaction - Don't try to use concurrently when building indexes by default (this was tripping things up when using with pg_cron) - Don't short circuit search for requests with ids (Fixes #159) - Fix for issue with pagination when sorting by columns with nulls (Fixes #161 Fixes #152) - Fixes issue where duplicate datetime,end_datetime index was being built. - Fix bug in pypgstac loader when using delsert option ### Added - Add trigger to detect duplicate configurations for name/collection combination in queryables - Add trigger to ensure collections added to queryables exist - Add tests for queryables triggers - Add more tests for different pagination scenarios ## [v0.7.0] ### Added - Reorganize code base to create clearer separation between pgstac sql code and pypgstac. - Move Python tooling to use hatch with all python project configuration in pyproject.toml - Rework testing framework to not rely on pypgstac or migrations. This allows to run tests on any code updates without creating a version first. If a new version has been staged, the tests will still run through all incremental migrations to make sure they pass as well. - Add pre-commit to run formatting as well as the tests appropriate for which files have changed. - Add a query queue to allow for deferred processing of steps that do not change the ability to get results, but enhance performance. The query queue allows to use pg_cron or similar to run tasks that are placed in the queue. - Modify triggers to allow the use of the query queue for building indexes, adding constraints that are used solely for constraint exclusion, and updating partition and collection spatial and temporal extents. The use of the queue is controlled by the new configuration parameter "use_queue" which can be set as the pgstac.use_queue GUC or by setting in the pgstac_settings table. - Reorganize how partitions are created and updated to maintain more metadata about partition extents and better tie the constraints to the actual temporal extent of a partition. - Add "partitions" view that shows stats about number of records, the partition range, constraint ranges, actual date range and spatial extent of each partition. - Add ability to automatically update the extent object on a collection using the partition metadata via triggers. This is controlled by the new configuration parameter "update_collection_extent" which can be set as the pgstac.update_collection_extent GUC or by setting in the pgstac_settings table. This can be combined with "use_queue" to defer the processing. - Add many new tests. - Migrations now make sure that all objects in the pgstac schema are owned by the pgstac_admin role. Functions marked as "SECURITY DEFINER" have been moved to the lower level functions responsible for creating/altering partitions and adding records to the search/search_wheres tables. This should open the door for approaches to using Row Level Security. - Allow pypgstac loader to load data on pgstac databases that have the same major version even if minor version differs. [162] () Cherry picked from . ### Fixed - Allow empty strings in datetime intervals - Set search_path and application_name upon connection rather than as kwargs for compatibility with RDS [156] () ## [v0.6.13] ### Fixed - Fix issue with sorting and paging where in some circumstances the aggregation of data changed the expected order ## [v0.6.12] ### Added - Add ability to merge enum, min, and max from queryables where collections have different values. - Add tooling in pypgstac and pgstac to add stac_extension definitions to the database. - Modify missing_queryables function to try to use stac_extension definitions to populate queryable definitions from the stac_extension schemas. - Add validate_constraints procedure - Add analyze_items procedure - Add check_pgstac_settings function to check system and pgstac settings. ### Fixed - Fix issue with upserts in the trigger for using the items_staging tables - Fix for generating token query for sorting. [152] () ## [v0.6.11] ### Fixed - update pypgstac requirements to support python 3.11 [142](https://github.com/stac-utils/pgstac/pull/142) - rename pgstac setting `default-filter-lang` to `default_filter_lang` to allow pgstac on postgresql>=14 ## [v0.6.10] ### Fixed - Makes sure that passing in a non-existing collection does not return a queryable object. ## [v0.6.9] ### Fixed - Set cursor_tuple_fraction to 1 in search function to let query planner know to expect the entire table result within the search function to be returned. The default cursor_tuple_fraction of .1 within that function was at times creating bad query plans leading to slow queries. ## [v0.6.8] ### Added - Add get_queryables function to return a composite queryables json for either a single collection (text), a list of collections(text[]), or for the full repository (null::text). - Add missing_queryables(collection text, tablesample int) function to help identify if there are any properties in a collection without entries in the queryables table. The tablesample parameter is an int <=100 that is the approximate percentage of the collection to scan to look for missing queryables rather than reading every item. - Add missing_queryables(tablesample int) function that scans all collections using a sample of records to identify missing queryables. ## [v0.6.7] ### Added - Add get_queryables function to return a composite queryables json for either a single collection (text), a list of collections(text[]), or for the full repository (null::text). - Add missing_queryables(collection text, tablesample int) function to help identify if there are any properties in a collection without entries in the queryables table. The tablesample parameter is an int <=100 that is the approximate percentage of the collection to scan to look for missing queryables rather than reading every item. - Add missing_queryables(tablesample int) function that scans all collections using a sample of records to identify missing queryables. ## [v0.6.6] ### Added - Add support for array operators in CQL2 (a_equals, a_contains, a_contained_by, a_overlaps). - Add check in loader to make sure that pypgstac and pgstac versions match before loading data [#123](https://github.com/stac-utils/pgstac/issues/123) ## [v0.6.5] ### Fixed - Fix for type casting when using the "in" operator [#122](https://github.com/stac-utils/pgstac/issues/122) - Fix failure of pypgstac load for large items [#121](https://github.com/stac-utils/pgstac/pull/121) ## [v0.6.4] ### Fixed - Fixed casts for numeric data when a property is not in the queryables table to use the type from the incoming json filter - Fixed issue loader grouping an unordered iterable by partition, speeding up loads of items with mixed partitions [#116](https://github.com/stac-utils/pgstac/pull/116) ## [v0.6.3] ### Fixed - Fixed content_hydrate argument ordering which caused incorrect behavior in database hydration [#115](https://github.com/stac-utils/pgstac/pull/115) ### Added - Skip partition updates when unnecessary, which can drastically improve large ingest performance into existing partitions. [#114](https://github.com/stac-utils/pgstac/pull/114) ## [v0.6.2] ### Fixed - Ensure special keys are not in content when loaded [#112](https://github.com/stac-utils/pgstac/pull/112/files) ## [v0.6.1] ### Fixed - Fix issue where using equality operator against an array was only comparing the first element of the array ## [v0.6.0] ### Fixed - Fix function signatures for transactional functions (delete_item etc) to make sure that they are marked as volatile - Fix function for getting start/end dates from a stac item ### Changed - Update hydration/dehydration logic to make sure that it matches hydration/dehydration in pypgstac - Update fields logic in pgstac to only use full paths and to match logic in stac-fastapi - Always include id and collection on features regardless of fields setting ### Added - Add tests to ensure that pgstac and pypgstac hydration logic is equivalent - Add conf item to search to allow returning results without hydrating. This allows an application using pgstac to shift the CPU load of rehydrating items from the database onto the application server. - Add "--dehydrated" option to loader to be able to load a dehydrated file (or iterable) of items such as would be output using pg_dump or postgresql copy. - Add "--chunksize" option to loader that can split the processing of an iterable or file into chunks of n records at a time ## [v0.5.1] ### Fixed ### Changed ### Added - Add conf item to search to allow returning results without hydrating. This allows an application using pgstac to shift the CPU load of rehydrating items from the database onto the application server. ## [v0.5.0] Version 0.5.0 is a major refactor of how data is stored. It is recommended to start a new database from scratch and to move data over rather than to use the inbuilt migration which will be very slow for larger amounts of data. ### Fixed ### Changed - The partition layout has been changed from being hardcoded to a partition to week to using nested partitions. The first level is by collection, for each collection, there is an attribute partition_trunc which can be set to NULL (no temporal partitions), month, or year. - CQL1 and Query Code have been refactored to translate to CQL2 to reduce duplicated code in query parsing. - Unused functions have been stripped from the project. - Pypgstac has been changed to use Fire rather than Typo. - Pypgstac has been changed to use Psycopg3 rather than Asyncpg to enable easier use as both sync and async. - Indexing has been reworked to eliminate indexes that from logs were not being used. The global json index on properties has been removed. Indexes on individual properties can be added either globally or per collection using the new queryables table. - Triggers for maintaining partitions have been updated to reduce lock contention and to reflect the new data layout. - The data pager which optimizes "order by datetime" searches has been updated to get time periods from the new partition layout and partition metadata. - Tests have been updated to reflect the many changes. ### Added - On ingest, the content in an item is compared to the metadata available at the collection level and duplicate information is stripped out (this is primarily data in the item_assets property). Logic is added in to merge this data back in on data usage. ## [v0.4.5] ### Fixed - Fixes support for using the intersects parameter at the base of a search (regression from changes in 0.4.4) - Fixes issue where results for a search on id returned [None] rather than [] (regression from changes in 0.4.4) ### Changed - Changes requirement for PostgreSQL to 13+, the triggers used to main partitions are not available to be used on partitions prior to 13 ([#90](https://github.com/stac-utils/pgstac/pull/90)) - Bump requirement for asyncpg to 0.25.0 ([#82](https://github.com/stac-utils/pgstac/pull/82)) ### Added - Added more tests. ## [v0.4.4] ### Added - Adds support for using ids, collections, datetime, bbox, and intersects parameters separated from the filter-lang (Fixes #85) - Previously use of these parameters was translated into cql-json and then to SQL, so was not available when using cql2-json - The deprecated query parameter is still only available when filter-lang is set to cql-json ### Changed - Add PLPGSQL for item lookups by id so that the query plan for the simple query can be cached - Use item_by_id function when looking up records used for paging filters - Add a short circuit to search to use item_by_id lookup when using the ids parameter - This short circuit avoids using the query cache for this simple case - Ordering when using the ids parameter is hard coded to return results in the same order as the array passed in (this avoids the overhead of full parsing and additional overhead to sort) ### Fixed - Fix to make sure that filtering on the search_wheres table leverages the functional index on the hash of the query rather than on the query itself. ## [v0.4.3] ### Fixed - Fix for optimization when using equals with json properties. Allow optimization for both "eq" and "=" (was only previously enabled for "eq") ## [v0.4.2] ### Changed - Add support for updated CQL2 spec to use timestamp or interval key ### Fixed - Fix for 0.3.4 -> 0.3.5 migration making sure that partitions get renamed correctly ## [v0.4.1] ### Changed - Update `typer` to 0.4.0 to avoid clashes with `click` ([#76](https://github.com/stac-utils/pgstac/pull/76)) ### Fixed - Fix logic in getting settings to make sure that filter-lang set on query is respected. ([#77](https://github.com/stac-utils/pgstac/pull/77)) - Fix for large queries in the query cache. ([#71](https://github.com/stac-utils/pgstac/pull/71)) ## [v0.4.0] ### Fixed - Fixes syntax for IN, BETWEEN, ISNULL, and NOT in CQL 1 ([#69](https://github.com/stac-utils/pgstac/pull/69)) ### Added - Adds support for modifying settings through pgstac_settings table and by passing in 'conf' object in search json to support AWS RDS where custom user configuration settings are not allowed and changing settings on the fly for a given query. - Adds support for CQL2-JSON ([#67](https://github.com/stac-utils/pgstac/pull/67)) - Adds tests for all examples in - filter-lang parameter controls which dialect of CQL to use - Adds 'default-filter-lang' setting to control what dialect to use when 'filter-lang' is not present - old style stac 'query' object and top level ids, collections, datetime, bbox, and intersects parameters are only available with cql-json ## [v0.3.4] ### Added - add `geometrysearch`, `geojsonsearch` and `xyzsearch` for optimized searches for tiled requets ([#39](https://github.com/stac-utils/pgstac/pull/39)) - add `create_items` and `upsert_items` methods for bulk insert ([#39](https://github.com/stac-utils/pgstac/pull/39)) ## [v0.3.3] ### Fixed - Fixed CQL term to be "id", not "ids" ([#46](https://github.com/stac-utils/pgstac/pull/46)) - Make sure featureCollection response has empty features `[]` not `null` ([#46](https://github.com/stac-utils/pgstac/pull/46)) - Fixed bugs for `sortby` and `pagination` ([#46](https://github.com/stac-utils/pgstac/pull/46)) - Make sure pgtap errors get caught in CI ([#46](https://github.com/stac-utils/pgstac/pull/46)) ## [v0.3.2] ## Fixed - Fixed CQL term to be "collections", not "collection" ([#43](https://github.com/stac-utils/pgstac/pull/43)) ## [v0.3.1] _TODO_ ## [v0.2.8] ### Added - Type hints to pypgstac that pass mypy checks ([#18](https://github.com/stac-utils/pgstac/pull/18)) ### Fixed - Fixed issue with pypgstac loads which caused some writes to fail ([#18](https://github.com/stac-utils/pgstac/pull/18)) [Unreleased]: https://github.com/stac-utils/pgstac/compare/v0.9.11...HEAD [v0.9.11]: https://github.com/stac-utils/pgstac/compare/v0.9.10...v0.9.11 [v0.9.10]: https://github.com/stac-utils/pgstac/compare/v0.9.9...v0.9.10 [v0.9.9]: https://github.com/stac-utils/pgstac/compare/v0.9.8...v0.9.9 [v0.9.8]: https://github.com/stac-utils/pgstac/compare/v0.9.7...v0.9.8 [v0.9.7]: https://github.com/stac-utils/pgstac/compare/v0.9.6...v0.9.7 [v0.9.6]: https://github.com/stac-utils/pgstac/compare/v0.9.5...v0.9.6 [v0.9.5]: https://github.com/stac-utils/pgstac/compare/v0.9.4...v0.9.5 [v0.9.4]: https://github.com/stac-utils/pgstac/compare/v0.9.3...v0.9.4 [v0.9.3]: https://github.com/stac-utils/pgstac/compare/v0.9.2...v0.9.3 [v0.9.2]: https://github.com/stac-utils/pgstac/compare/v0.9.1...v0.9.2 [v0.9.1]: https://github.com/stac-utils/pgstac/compare/v0.9.0...v0.9.1 [v0.9.0]: https://github.com/stac-utils/pgstac/compare/v0.8.5...v0.9.0 [v0.8.5]: https://github.com/stac-utils/pgstac/compare/v0.8.4...v0.8.5 [v0.8.4]: https://github.com/stac-utils/pgstac/compare/v0.8.3...v0.8.4 [v0.8.3]: https://github.com/stac-utils/pgstac/compare/v0.8.2...v0.8.3 [v0.8.2]: https://github.com/stac-utils/pgstac/compare/v0.8.1...v0.8.2 [v0.8.1]: https://github.com/stac-utils/pgstac/compare/v0.8.0...v0.8.1 [v0.8.0]: https://github.com/stac-utils/pgstac/compare/v0.7.10...v0.8.0 [v0.7.10]: https://github.com/stac-utils/pgstac/compare/v0.7.9...v0.7.10 [v0.7.9]: https://github.com/stac-utils/pgstac/compare/v0.7.8...v0.7.9 [v0.7.8]: https://github.com/stac-utils/pgstac/compare/v0.7.7...v0.7.8 [v0.7.7]: https://github.com/stac-utils/pgstac/compare/v0.7.6...v0.7.7 [v0.7.6]: https://github.com/stac-utils/pgstac/compare/v0.7.5...v0.7.6 [v0.7.5]: https://github.com/stac-utils/pgstac/compare/v0.7.4...v0.7.5 [v0.7.4]: https://github.com/stac-utils/pgstac/compare/v0.7.3...v0.7.4 [v0.7.3]: https://github.com/stac-utils/pgstac/compare/v0.7.2...v0.7.3 [v0.7.2]: https://github.com/stac-utils/pgstac/compare/v0.7.1...v0.7.2 [v0.7.1]: https://github.com/stac-utils/pgstac/compare/v0.7.0...v0.7.1 [v0.7.0]: https://github.com/stac-utils/pgstac/compare/v0.6.13...v0.7.0 [v0.6.13]: https://github.com/stac-utils/pgstac/compare/v0.6.12...v0.6.13 [v0.6.12]: https://github.com/stac-utils/pgstac/compare/v0.6.11...v0.6.12 [v0.6.11]: https://github.com/stac-utils/pgstac/compare/v0.6.10...v0.6.11 [v0.6.10]: https://github.com/stac-utils/pgstac/compare/v0.6.9...v0.6.10 [v0.6.9]: https://github.com/stac-utils/pgstac/compare/v0.6.8...v0.6.9 [v0.6.8]: https://github.com/stac-utils/pgstac/compare/v0.6.7...v0.6.8 [v0.6.7]: https://github.com/stac-utils/pgstac/compare/v0.6.6...v0.6.7 [v0.6.6]: https://github.com/stac-utils/pgstac/compare/v0.6.5...v0.6.6 [v0.6.5]: https://github.com/stac-utils/pgstac/compare/v0.6.4...v0.6.5 [v0.6.4]: https://github.com/stac-utils/pgstac/compare/v0.6.3...v0.6.4 [v0.6.3]: https://github.com/stac-utils/pgstac/compare/v0.6.2...v0.6.3 [v0.6.2]: https://github.com/stac-utils/pgstac/compare/v0.6.1...v0.6.2 [v0.6.1]: https://github.com/stac-utils/pgstac/compare/v0.6.0...v0.6.1 [v0.6.0]: https://github.com/stac-utils/pgstac/compare/v0.5.1...v0.6.0 [v0.5.1]: https://github.com/stac-utils/pgstac/compare/v0.5.0...v0.5.1 [v0.5.0]: https://github.com/stac-utils/pgstac/compare/v0.4.5...v0.5.0 [v0.4.5]: https://github.com/stac-utils/pgstac/compare/v0.4.4...v0.4.5 [v0.4.4]: https://github.com/stac-utils/pgstac/compare/v0.4.3...v0.4.4 [v0.4.3]: https://github.com/stac-utils/pgstac/compare/v0.4.2...v0.4.3 [v0.4.2]: https://github.com/stac-utils/pgstac/compare/v0.4.1...v0.4.2 [v0.4.1]: https://github.com/stac-utils/pgstac/compare/v0.4.0...v0.4.1 [v0.4.0]: https://github.com/stac-utils/pgstac/compare/v0.3.4...v0.4.0 [v0.3.4]: https://github.com/stac-utils/pgstac/compare/v0.3.3...v0.3.4 [v0.3.3]: https://github.com/stac-utils/pgstac/compare/v0.3.2...v0.3.3 [v0.3.2]: https://github.com/stac-utils/pgstac/compare/v0.3.1...v0.3.2 [v0.3.1]: https://github.com/stac-utils/pgstac/compare/v0.3.0...v0.3.1 [v0.2.8]: https://github.com/stac-utils/pgstac/compare/ff02c9cee7bbb0a2de21530b0aeb34e823f2e95c...v0.2.8 ================================================ FILE: CLAUDE.md ================================================ # PgSTAC Development Instructions ## Project Overview PgSTAC is a PostgreSQL extension (SQL functions + schema) for Spatio-Temporal Asset Catalogs (STAC), paired with pypgstac, a Python package for database migrations and bulk data ingestion. - **Repository**: stac-utils/pgstac - **License**: MIT - **Docs**: https://stac-utils.github.io/pgstac/ ## Architecture ``` src/pgstac/sql/ ← ALL SQL source files (edit ONLY here) src/pgstac/pgstac.sql ← Assembled output (DO NOT edit directly) src/pgstac/migrations/ ← Base + incremental migration files src/pgstac/tests/ ← PGTap and basic SQL tests src/pypgstac/src/pypgstac/ ← Python package source src/pypgstac/tests/ ← pytest tests scripts/ ← Host-facing entrypoint scripts scripts/container-scripts/ ← Scripts copied into the pypgstac container image ``` ### Documentation Files - **`CHANGELOG.md`** — the single source of truth for release notes - **`docs/src/release-notes.md`** — a **manual copy** of `CHANGELOG.md`, served by mkdocs. Keep them identical; update both when changing either. ### Database Roles - **pgstac_admin** – schema owner, migrations - **pgstac_ingest** – read/write, execute functions - **pgstac_read** – SELECT only ## Critical Rules ### SQL Changes **ONLY edit files in `src/pgstac/sql/`**. Never edit `pgstac.sql` directly — it is assembled by concatenating all `sql/*.sql` files in alphabetical order during `stageversion`. File execution order (alphabetical by prefix): `000_idempotent_pre` → `001_core` → `001a_jsonutils` → `001s_stacutils` → `002_collections` → `002a_queryables` → `002b_cql` → `003a_items` → `003b_partitions` → `004_search` → `004a_collectionsearch` → `005_tileutils` → `006_tilesearch` → `997_maintenance` → `998_idempotent_post` → `999_version` Prefix ranges: `000-001` setup/core, `002` collections, `003` items/partitions, `004-006` search/tiles, `997-998` maintenance/post, `999` version (auto-generated). ### Idempotency `000_idempotent_pre.sql` and `998_idempotent_post.sql` are included in both base installs and incremental migrations. Use `IF NOT EXISTS`, `CREATE OR REPLACE`, `ON CONFLICT DO NOTHING`. ### Partitioning Items partitioned by `LIST(collection)`, optionally sub-partitioned by `RANGE(datetime)` (year/month via `collections.partition_trunc`). Naming: `_items_{key}[_{YYYY|YYYYMM}]`. Key functions: `check_partition()` (create/update), `update_partition_stats()` (recalculate constraints), `partition_sys_meta` (live VIEW — always current), `partitions` (MATERIALIZED VIEW — stale between refreshes). ### Search Path PgSTAC installs into the `pgstac` schema. All connections must have `search_path` set to `pgstac, public`. ### pg_dump / pg_restore Compatibility PgSTAC functions reference PostGIS functions (e.g. `st_makeenvelope`, `st_geomfromgeojson`) **without schema qualification** because PostGIS may be installed in either `public` or `postgis` schema. `pg_dump` clears `search_path` during restore, breaking these references. **Rules to maintain dump/restore compatibility:** - **Do NOT schema-qualify PostGIS function calls** in PgSTAC SQL - **Avoid cross-function dependencies in SQL functions used by GENERATED columns** — pg_dump orders functions alphabetically, so `func_a` calling `func_b` may be created before `func_b` exists. Inline the logic instead. - Use `pgstac_restore` (via `scripts/container-scripts/pgstac_restore` in the image) to restore dumps — it installs a temporary event trigger that sets the correct `search_path` before each DDL command - Test with `scripts/test --pgdump` ## Development Workflow ### Setup ```bash scripts/setup # Build Docker images, start database scripts/server # Start database (use --detach for background) ``` ### Running Tests ```bash scripts/test # All test suites scripts/test --pypgstac # pytest only scripts/test --pgtap # PGTap SQL tests scripts/test --basicsql # SQL output comparison tests scripts/test --migrations # Full migration chain test scripts/test --formatting # ruff + ty scripts/test --pgdump # pg_dump/pg_restore round-trip test ``` All tests run inside Docker via `scripts/runinpypgstac`. Use `--build` to rebuild images first. ### Docker Architecture - **pgstac** container: PostgreSQL 17 + PostGIS 3 + extensions, port 5439→5432 - **pypgstac** container: Python + Rust build tools, runs scripts - Credentials: `username` / `password`, database: `postgis` ## Migration Process ### Creating Migrations (Release) ```bash scripts/stageversion 0.9.11 ``` This runs inside Docker and: 1. Removes old `*unreleased*` migration files 2. Writes `SELECT set_version('0.9.11');` to `999_version.sql` 3. Concatenates all `sql/*.sql` → `migrations/pgstac.0.9.11.sql` (base migration) 4. Copies the base migration to `pgstac.sql` 5. Updates `version.py` and `pyproject.toml` version strings 6. Runs `makemigration -f 0.9.10 -t 0.9.11` to generate incremental migration ### How makemigration Works `makemigration` (copied from `scripts/container-scripts/makemigration` into the image) generates incremental migrations by diffing schemas: 1. Creates two temp databases: `migra_from`, `migra_to` 2. Loads old base migration into `migra_from` 3. Loads new base migration into `migra_to` 4. Runs `migra --schema pgstac --unsafe` to calculate the SQL diff 5. Wraps the diff with `000_idempotent_pre.sql`, `998_idempotent_post.sql`, and `set_version()` 6. Output: `migrations/pgstac.0.9.10-0.9.11.sql` **Important**: The generated migration is created with a `.staged` suffix. You MUST: 1. Review the `.staged` file for correctness 2. Remove the `.staged` suffix to enable it 3. Run `scripts/test --migrations` to validate ### Running Migrations ```bash pypgstac migrate # Migrate to current pypgstac version pypgstac migrate --toversion 0.9.10 # Migrate to specific version ``` The `Migrate` class (in `migrate.py`) builds a directed graph of all available migration files and uses BFS to find the shortest path from the current DB version to the target. ## Testing Details ### Test Database Setup Tests create `pgstac_test_db_template` from `pgstac.sql`, then clone it per test suite: - `pgstac_test_pgtap` – PGTap tests - `pgstac_test_basicsql` – basic SQL tests - `pgstac_test_pypgstac` – pytest (function-scoped fixture creates fresh DB per test) ### Test Types 1. **PGTap**: SQL assertions in `src/pgstac/tests/pgtap.sql` 2. **Basic SQL**: `.sql` files in `src/pgstac/tests/basic/`, output compared to `.sql.out` 3. **Pytest**: `src/pypgstac/tests/test_load.py`, `test_benchmark.py`, `test_queryables.py`, `hydration/` 4. **Migration**: Installs v0.3.0, migrates to latest, runs all test suites against migrated DB 5. **pg_dump**: Dumps a database with sample data, restores via `pgstac_restore`, verifies counts match ### Pytest Fixtures (conftest.py) - `db` – function-scoped `PgstacDB` connected to fresh test DB - `loader` – `Loader(db)` instance ## PR Checklist 1. Changes only in `src/pgstac/sql/` for SQL, `src/pypgstac/` for Python 2. Tests added if appropriate 3. `CHANGELOG.md` updated under `## [UNRELEASED]` 4. `docs/src/release-notes.md` updated to match `CHANGELOG.md` (they must stay identical) 5. Docs updated if needed 6. All tests pass: `scripts/test` (or `scripts/runinpypgstac --build test --pypgstac`) ## Release Checklist 1. `scripts/stageversion VERSION` 2. Review `.staged` migration, remove suffix 3. `scripts/test --migrations` 4. Move CHANGELOG "Unreleased" → new version 5. Copy updated `CHANGELOG.md` to `docs/src/release-notes.md` (keep identical) 6. Create PR, merge 7. `git tag vVERSION && git push origin vVERSION` 8. CI publishes to PyPI + ghcr.io ## Common Patterns ### Adding a new SQL function 1. Edit the appropriate file in `src/pgstac/sql/` (use `CREATE OR REPLACE FUNCTION`) 2. Add `SECURITY DEFINER` if the function modifies tables 3. Grant execute in `998_idempotent_post.sql` if needed 4. Add PGTap or basic SQL tests ### Adding a new queryable ```sql INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('prop_name', '{"$ref": "..."}', 'to_int', 'BTREE') ON CONFLICT DO NOTHING; ``` ### Loading test data ```bash scripts/runinpypgstac --build loadsampledata ``` ================================================ FILE: CONTRIBUTING.md ================================================ # Development - Contributing PgSTAC uses a dockerized development environment. Local-only planning notes now live under `.plans/` and are intentionally gitignored. If you keep local execution settings, start from `.env.example` and write overrides to `.env`. To build the docker images and set up the test database, use: ```bash scripts/setup ``` To bring up the development database: ``` scripts/server ``` To run tests, use: ```bash scripts/test ``` To set up pre-commit using the project uv workflow: ```bash uv tool install pre-commit==3.5.0 pre-commit install ``` Useful options: ```bash scripts/test --fast scripts/test --pypgstac scripts/test --build-policy always ``` `scripts/test` defaults to `PGSTAC_BUILD_POLICY=always` so the container image reflects your current checkout. If you intentionally want to reuse an existing image, set `PGSTAC_BUILD_POLICY=missing` or pass `--build-policy missing`. To rebuild docker images: ```bash scripts/update ``` Container-only helper scripts now live in `scripts/container-scripts/` and are copied into the `pypgstac` image during build. Top-level `scripts/` remain the host-facing entrypoint surface. To drop into a console, use ```bash scripts/console ``` To drop into a psql console on the database container, use: ```bash scripts/console --db ``` To run migrations on the development database, use ```bash scripts/migrate ``` To stage code and configurations and create template migrations for a version release, use ```bash scripts/stageversion [version] ``` To generate only the incremental migration, use: ```bash scripts/makemigration --from 0.9.10 --to 0.9.11 ``` Examples: ``` scripts/stageversion 0.2.8 ``` This will create a base migration for the new version and will create incremental migrations between any existing base migrations. The incremental migrations that are automatically generated by this script will have the extension ".staged" on the file. You must manually review (and make any modifications necessary) this file and remove the ".staged" extension to enable the migration. ### Making Changes to SQL All changes to SQL should only be made in the `/src/pgstac/sql` directory. SQL Files will be run in alphabetical order. ### Adding Tests There are three different types of tests within the project: (1) pgTap tests, (2) basic SQL tests, and (3) PyPgSTAC tests. PgSTAC tests can be written using PGTap or basic SQL output comparisons. Additional testing is available using PyTest in the PyPgSTAC module. Tests can be run using the `scripts/test` command. PGTap tests can be written using [PGTap](https://pgtap.org/) syntax. Tests should be added to the `/src/pgstac/tests/pgtap` directory. Any new SQL files added to this directory must be added to `/src/pgstac/tests/pgtap.sql`. The Basic SQL tests will run any file ending in '.sql' in the `/src/pgstac/tests/basic` directory and will compare the exact results to the corresponding '.sql.out' file. PyPgSTAC tests are pytest tests, and they are located in `/src/pypgstac/tests` All tests can be found in tests/pgtap.sql and are run using `scripts/test`. Individual tests can be run with any combination of the following flags `--formatting --basicsql --pgtap --migrations --pypgstac`. The `--formatting` suite runs Ruff lint/format checks and Ty type checks. If pre-commit is installed, tests will be run on commit based on which files have changed. ### To make a PR 1) Make any changes. 2) Make sure there are tests if appropriate. 3) Update Changelog using "### Unreleased" as the version. 4) Make any changes necessary to the docs. 5) Ensure all tests pass (pre-commit will take care of this if installed and the tests will also run on CI) 6) Create PR against the "main" branch. ### Release Process 1) Run "scripts/stageversion VERSION" (where version is the next version using semantic versioning ie 0.7.0 2) Check the incremental migration created in the /src/pgstac/migrations file with the .staged extension to make sure that the generated SQL looks appropriate. 3) Run the tests against the incremental migrations "scripts/test --migrations" 4) Move any "Unreleased" changes in the CHANGELOG.md to the new version. 5) Open a PR for the version change. 6) Once the PR has been merged, start the release process. 7) Create a git tag `git tag v0.2.8` using new version number 8) Push the git tag `git push origin v0.2.8` 9) The CI process will push pypgstac to PyPi, create a docker image on ghcr.io, and create a release on github. ### Get Involved Issues and pull requests are more than welcome: https://github.com/stac-utils/pgstac/issues ### A Note on Hydration and Dehydration Dehydration refers to stripping redundant attributes of STAC items when storing them within the database. For many collections, dehydration saves a significant amount of memory. Rehydration is the process of adding the stripped attributes back to the STAC items, such as during the export of an STAC collection or the response to a search query. PgSTAC, a versatile tool, is designed to seamlessly integrate with PyPgSTAC or alternative backends. This flexibility allows for direct calls for both rehydration and dehydration, giving developers and technical users a sense of control over the process. Hydration and dehydration are de-facto settings that users can not opt out of. In the future, we may provide a configuration for use cases where the size benefits do not justify the added complexity. ================================================ FILE: LICENSE ================================================ MIT License Copyright (c) Microsoft Corporation and other contributors. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE ================================================ FILE: README.md ================================================

PostgreSQL schema and functions for Spatio-Temporal Asset Catalog (STAC)

Test Package version License

--- **Documentation**: https://stac-utils.github.io/pgstac/ **Source Code**: https://github.com/stac-utils/pgstac --- **PgSTAC** is a set of SQL functions and schema to build highly performant databases for Spatio-Temporal Asset Catalogs ([STAC](https://stacspec.org/)). The project also provides **pypgstac** (a Python module) to help with database migrations and document ingestion (collections and items). PgSTAC provides functionality for STAC Filters, CQL2 search, and utilities to help manage the indexing and partitioning of STAC Collections and Items. PgSTAC is used in production to scale to hundreds of millions of STAC items. PgSTAC implements core data models and functions to provide a STAC API from a PostgreSQL database. PgSTAC is entirely within the database and does not provide an HTTP-facing API. The [STAC FastAPI](https://github.com/stac-utils/stac-fastapi) PgSTAC backend and [Franklin](https://github.com/azavea/franklin) can be used to expose a PgSTAC catalog. Integrating PgSTAC with any other language with PostgreSQL drivers is also possible. PgSTAC Documentation: https://stac-utils.github.io/pgstac/pgstac pyPgSTAC Documentation: https://stac-utils.github.io/pgstac/pypgstac ## Project structure ``` / ├── src/pypgstac - pyPgSTAC python module ├── src/pypgstac/tests/ - pyPgSTAC tests ├── scripts/ - scripts to set up the environment, create migrations, and run tests ├── src/pgstac/sql/ - PgSTAC SQL code ├── src/pgstac/migrations/ - Migrations for incremental upgrades └── src/pgstac/tests/ - test suite ``` ## Contribution & Development See [CONTRIBUTING.md](https://github.com//stac-utils/pgstac/blob/master/CONTRIBUTING.md) ## License See [LICENSE](https://github.com//stac-utils/pgstac/blob/master/LICENSE) ## Authors See [contributors](https://github.com/stac-utils/pgstac/graphs/contributors) for a listing of individual contributors. ## Changes See [CHANGELOG.md](https://github.com/stac-utils/pgstac/blob/master/CHANGELOG.md). ================================================ FILE: docker/pgstac/Dockerfile ================================================ # syntax=docker/dockerfile:1.7 ARG PG_MAJOR=17 ARG POSTGIS_MAJOR=3 ARG PG_TLE_VERSION=1.5.2 ARG DEBIAN_SUITE=trixie # Base postgres image that pgstac can be installed onto FROM postgres:${PG_MAJOR}-${DEBIAN_SUITE} AS pgstacbase ARG POSTGIS_MAJOR ARG PG_MAJOR ARG PG_TLE_VERSION RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ --mount=type=cache,target=/root/.cache/git,sharing=locked \ apt-get update \ && apt-get install -y --no-install-recommends \ postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR \ postgresql-$PG_MAJOR-postgis-$POSTGIS_MAJOR-scripts \ postgresql-$PG_MAJOR-pgtap \ postgresql-$PG_MAJOR-plpgsql-check \ postgresql-$PG_MAJOR-partman \ postgresql-server-dev-$PG_MAJOR \ build-essential \ ca-certificates \ curl \ git \ flex \ bison \ libkrb5-dev \ && GIT_TERMINAL_PROMPT=0 git clone --branch v${PG_TLE_VERSION} --depth 1 https://github.com/aws/pg_tle.git /tmp/pg_tle \ && make -C /tmp/pg_tle \ && make -C /tmp/pg_tle install \ && rm -rf /tmp/pg_tle \ && sed -i "s/^#shared_preload_libraries = .*/shared_preload_libraries = 'pg_tle'/" /usr/share/postgresql/$PG_MAJOR/postgresql.conf.sample \ && sed -i "s/^#shared_preload_libraries = .*/shared_preload_libraries = 'pg_tle'/" /usr/share/postgresql/postgresql.conf.sample \ && apt-get purge -y --auto-remove \ postgresql-server-dev-$PG_MAJOR \ build-essential \ curl \ git \ flex \ bison \ libkrb5-dev \ && apt-get clean && apt-get -y autoremove \ && rm -rf /var/lib/apt/lists/* # The pgstacbase image with latest version of pgstac installed FROM pgstacbase AS pgstac WORKDIR /docker-entrypoint-initdb.d COPY docker/pgstac/dbinit/pgstac.sh 990_pgstac.sh COPY src/pgstac/pgstac.sql 999_pgstac.sql ================================================ FILE: docker/pgstac/dbinit/pgstac.sh ================================================ SYSMEM=$(cat /proc/meminfo | grep -i 'memtotal' | grep -o '[[:digit:]]*') SHARED_BUFFERS=$(( $SYSMEM/4 )) EFFECTIVE_CACHE_SIZE=$(( $SYSMEM*3/4 )) MAINTENANCE_WORK_MEM=$(( $SYSMEM/8 )) WORK_MEM=$(( $SHARED_BUFFERS/50 )) psql -X -q -v ON_ERROR_STOP=1 </tmp/requirements.txt \ && uv pip install --system -r /tmp/requirements.txt COPY scripts/container-scripts /opt/pgstac/container-scripts COPY src/pypgstac /opt/src/pypgstac COPY src/pgstac /opt/src/pgstac WORKDIR /opt/src/pypgstac RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ uv pip install --system -e . \ && rm -rf /usr/local/cargo/registry RUN addgroup --gid 1000 user && \ adduser --uid 1000 --gid 1000 --disabled-password --gecos "" --home /home/user user && \ chown -R user:user /opt/src/pypgstac /opt/src/pgstac USER user # Optional runtime-optimized image: no build toolchain, only pypgstac package + runtime deps. FROM python:3.13-slim-trixie AS pypgstac-runtime ENV PYTHONWRITEBYTECODE=1 ENV PYTHONBUFFERED=1 ENV UV_BREAK_SYSTEM_PACKAGES=1 ENV PATH="/opt/pgstac/container-scripts:$PATH" ENV UV_CACHE_DIR=/root/.cache/uv ARG PG_MAJOR=17 RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ --mount=type=cache,target=/var/lib/apt/lists,sharing=locked \ --mount=type=cache,target=/root/.cache/uv,sharing=locked \ apt-get update \ && apt-get install -y --no-install-recommends \ ca-certificates \ curl \ postgresql-client-${PG_MAJOR} \ && curl -LsSf https://astral.sh/uv/install.sh | UV_INSTALL_DIR=/usr/local/bin sh \ && apt-get clean && rm -rf /var/lib/apt/lists/* COPY ./src/pypgstac/pyproject.toml /tmp/pyproject.toml RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ uv pip compile /tmp/pyproject.toml \ --extra psycopg \ --extra migrations \ >/tmp/requirements.txt \ && uv pip install --system -r /tmp/requirements.txt COPY scripts/container-scripts /opt/pgstac/container-scripts COPY src/pypgstac /opt/src/pypgstac COPY src/pgstac /opt/src/pgstac WORKDIR /opt/src/pypgstac RUN --mount=type=cache,target=/root/.cache/uv,sharing=locked \ uv pip install --system . ================================================ FILE: docker-compose.yml ================================================ services: pgstac: image: pgstac env_file: - .env build: context: . network: host dockerfile: docker/pgstac/Dockerfile target: pgstac platform: linux/amd64 environment: - POSTGRES_USER=${POSTGRES_USER:-username} - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-password} - POSTGRES_DB=${POSTGRES_DB:-postgis} - PGHOST=/var/run/postgresql - PGPORT=5432 - PGUSER=${PGUSER:-username} - PGPASSWORD=${PGPASSWORD:-password} - PGDATABASE=${PGDATABASE:-postgis} ports: - "${PGPORT:-5439}:5432" volumes: - pgstac-pgdata:/var/lib/postgresql/data command: postgres healthcheck: test: ["CMD-SHELL", "pg_isready -h /var/run/postgresql -p 5432 -U ${PGUSER:-username} -d ${PGDATABASE:-postgis}"] interval: 5s timeout: 5s retries: 12 start_period: 5s pypgstac: image: pypgstac env_file: - .env build: context: . network: host dockerfile: docker/pypgstac/Dockerfile target: pypgstac platform: linux/amd64 environment: - PGHOST=pgstac - PGPORT=5432 - PGUSER=${PGUSER:-username} - PGPASSWORD=${PGPASSWORD:-password} - PGDATABASE=${PGDATABASE:-postgis} depends_on: pgstac: condition: service_healthy volumes: pgstac-pgdata: ================================================ FILE: docs/mkdocs.yml ================================================ site_name: pgstac site_description: PostgreSQL schema and functions for Spatio-Temporal Asset Catalog (STAC). docs_dir: 'src' site_dir: 'build' repo_name: "stac-utils/pgstac" repo_url: "https://github.com/stac-utils/pgstac" edit_uri: "blob/master/docs/src" site_url: "https://stac-utils.github.io/pgstac/" extra: social: - icon: "fontawesome/brands/github" link: "https://github.com/stac-utils" - icon: "fontawesome/brands/twitter" link: "https://twitter.com/STACspec" nav: - Home: "index.md" - PgSTAC: "pgstac.md" - pyPgSTAC: "pypgstac.md" - Performance: - item_size_analysis.ipynb - Development - Contributing: "contributing.md" - Release Notes: "release-notes.md" plugins: - search - mkdocs-jupyter: include_source: True include_requirejs: True execute: True show_input: False theme: name: material palette: primary: indigo scheme: default # https://github.com/kylebarron/cogeo-mosaic/blob/mkdocs/mkdocs.yml#L50-L75 markdown_extensions: - admonition - attr_list - codehilite: guess_lang: false - def_list - footnotes - pymdownx.arithmatex - pymdownx.betterem - pymdownx.caret: insert: false - pymdownx.details - pymdownx.emoji - pymdownx.escapeall: hardbreak: true nbsp: true - pymdownx.magiclink: hide_protocol: true repo_url_shortener: true - pymdownx.smartsymbols - pymdownx.superfences - pymdownx.tasklist: custom_checkbox: true - pymdownx.tilde - toc: permalink: true ================================================ FILE: docs/src/benchmark.json ================================================ { "machine_info": { "node": "quercus", "processor": "x86_64", "machine": "x86_64", "python_compiler": "GCC 13.2.0", "python_implementation": "CPython", "python_implementation_version": "3.12.3", "python_version": "3.12.3", "python_build": [ "main", "Nov 6 2024 18:32:19" ], "release": "6.8.0-51-generic", "system": "Linux", "cpu": { "python_version": "3.12.3.final.0 (64 bit)", "cpuinfo_version": [ 9, 0, 0 ], "cpuinfo_version_string": "9.0.0", "arch": "X86_64", "bits": 64, "count": 16, "arch_string_raw": "x86_64", "vendor_id_raw": "AuthenticAMD", "brand_raw": "AMD Ryzen 7 7840U w/ Radeon 780M Graphics", "hz_advertised_friendly": "1.7866 GHz", "hz_actual_friendly": "1.7866 GHz", "hz_advertised": [ 1786629000, 0 ], "hz_actual": [ 1786629000, 0 ], "stepping": 1, "model": 116, "family": 25, "flags": [ "3dnowext", "3dnowprefetch", "abm", "adx", "aes", "amd_lbr_v2", "aperfmperf", "apic", "arat", "avx", "avx2", "avx512_bf16", "avx512_bitalg", "avx512_vbmi2", "avx512_vnni", "avx512_vpopcntdq", "avx512bitalg", "avx512bw", "avx512cd", "avx512dq", "avx512f", "avx512ifma", "avx512vbmi", "avx512vbmi2", "avx512vl", "avx512vnni", "avx512vpopcntdq", "bmi1", "bmi2", "bpext", "cat_l3", "cdp_l3", "clflush", "clflushopt", "clwb", "clzero", "cmov", "cmp_legacy", "constant_tsc", "cpb", "cppc", "cpuid", "cqm", "cqm_llc", "cqm_mbm_local", "cqm_mbm_total", "cqm_occup_llc", "cr8_legacy", "cx16", "cx8", "dbx", "de", "decodeassists", "erms", "extapic", "extd_apicid", "f16c", "flush_l1d", "flushbyasid", "fma", "fpu", "fsgsbase", "fsrm", "fxsr", "fxsr_opt", "gfni", "ht", "hw_pstate", "ibpb", "ibrs", "ibrs_enhanced", "ibs", "invpcid", "irperf", "lahf_lm", "lbrv", "lm", "mba", "mca", "mce", "misalignsse", "mmx", "mmxext", "monitor", "movbe", "msr", "mtrr", "mwaitx", "nonstop_tsc", "nopl", "npt", "nrip_save", "nx", "ospke", "osvw", "osxsave", "overflow_recov", "pae", "pat", "pausefilter", "pci_l2i", "pclmulqdq", "pdpe1gb", "perfctr_core", "perfctr_llc", "perfctr_nb", "perfmon_v2", "pfthreshold", "pge", "pku", "pni", "popcnt", "pqe", "pqm", "pse", "pse36", "rapl", "rdpid", "rdpru", "rdrand", "rdrnd", "rdseed", "rdt_a", "rdtscp", "rep_good", "sep", "sha", "sha_ni", "skinit", "smap", "smca", "smep", "ssbd", "sse", "sse2", "sse4_1", "sse4_2", "sse4a", "ssse3", "stibp", "succor", "svm", "svm_lock", "syscall", "tce", "topoext", "tsc", "tsc_scale", "umip", "user_shstk", "v_spec_ctrl", "v_vmsave_vmload", "vaes", "vgif", "vmcb_clean", "vme", "vmmcall", "vnmi", "vpclmulqdq", "wbnoinvd", "wdt", "x2apic", "x2avic", "xgetbv1", "xsave", "xsavec", "xsaveerptr", "xsaveopt", "xsaves" ], "l3_cache_size": 1048576, "l2_cache_size": 8388608, "l1_data_cache_size": 262144, "l1_instruction_cache_size": 262144, "l2_cache_line_size": 1024, "l2_cache_associativity": 6 } }, "commit_info": { "id": "6da165b7c4f6da321d0b58bbb78da2887763a0d8", "time": "2024-12-09T10:50:39-06:00", "author_time": "2024-12-09T10:50:39-06:00", "dirty": true, "project": "pgstac", "branch": "feat/search-benchmarks" }, "benchmarks": [ { "group": "xyzsearch", "name": "test1[3-0.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-0.5]", "params": { "zoom": 3, "item_width": 0.5 }, "param": "3-0.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 4.781100236999919, "max": 4.923354106998886, "mean": 4.84395678366612, "stddev": 0.07255507461212232, "rounds": 3, "median": 4.827416006999556, "iqr": 0.10669040249922546, "q1": 4.792679179499828, "q3": 4.8993695819990535, "iqr_outliers": 0, "stddev_outliers": 1, "outliers": "1;0", "ld15iqr": 4.781100236999919, "hd15iqr": 4.923354106998886, "ops": 0.20644279969053644, "total": 14.53187035099836, "data": [ 4.781100236999919, 4.827416006999556, 4.923354106998886 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[3-0.75]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-0.75]", "params": { "zoom": 3, "item_width": 0.75 }, "param": "3-0.75", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 2.147751219003112, "max": 2.1683643510004913, "mean": 2.158085888334123, "stddev": 0.010306680943796004, "rounds": 3, "median": 2.158142094998766, "iqr": 0.015459848998034431, "q1": 2.1503489380020255, "q3": 2.16580878700006, "iqr_outliers": 0, "stddev_outliers": 1, "outliers": "1;0", "ld15iqr": 2.147751219003112, "hd15iqr": 2.1683643510004913, "ops": 0.4633735874024566, "total": 6.474257665002369, "data": [ 2.1683643510004913, 2.158142094998766, 2.147751219003112 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[3-1]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-1]", "params": { "zoom": 3, "item_width": 1 }, "param": "3-1", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 1.1977020270023786, "max": 1.2047663939993072, "mean": 1.2008456296668253, "stddev": 0.003595734342317445, "rounds": 3, "median": 1.2000684679987899, "iqr": 0.005298275247696438, "q1": 1.1982936372514814, "q3": 1.2035919124991779, "iqr_outliers": 0, "stddev_outliers": 1, "outliers": "1;0", "ld15iqr": 1.1977020270023786, "hd15iqr": 1.2047663939993072, "ops": 0.8327465040426971, "total": 3.6025368890004756, "data": [ 1.2047663939993072, 1.1977020270023786, 1.2000684679987899 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[3-1.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-1.5]", "params": { "zoom": 3, "item_width": 1.5 }, "param": "3-1.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.5610035069985315, "max": 0.5700876880000578, "mean": 0.5648575563330572, "stddev": 0.004695826663797881, "rounds": 3, "median": 0.5634814740005822, "iqr": 0.006813135751144728, "q1": 0.5616229987490442, "q3": 0.5684361345001889, "iqr_outliers": 0, "stddev_outliers": 1, "outliers": "1;0", "ld15iqr": 0.5610035069985315, "hd15iqr": 0.5700876880000578, "ops": 1.7703578340914847, "total": 1.6945726689991716, "data": [ 0.5610035069985315, 0.5634814740005822, 0.5700876880000578 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[3-2]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-2]", "params": { "zoom": 3, "item_width": 2 }, "param": "3-2", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.3394572959987272, "max": 0.3474930239972309, "mean": 0.3423380219983301, "stddev": 0.0035336363516854353, "rounds": 4, "median": 0.3412008839986811, "iqr": 0.004039818997625844, "q1": 0.34031811249951716, "q3": 0.344357931497143, "iqr_outliers": 0, "stddev_outliers": 1, "outliers": "1;0", "ld15iqr": 0.3394572959987272, "hd15iqr": 0.3474930239972309, "ops": 2.921089495588889, "total": 1.3693520879933203, "data": [ 0.3394572959987272, 0.3474930239972309, 0.3412228389970551, 0.3411789290003071 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[3-3]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-3]", "params": { "zoom": 3, "item_width": 3 }, "param": "3-3", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.1616739819983195, "max": 0.1732538390024274, "mean": 0.1683614392864651, "stddev": 0.004645194475107718, "rounds": 7, "median": 0.16951855400111526, "iqr": 0.007849024001188809, "q1": 0.16395422725054232, "q3": 0.17180325125173113, "iqr_outliers": 0, "stddev_outliers": 3, "outliers": "3;0", "ld15iqr": 0.1616739819983195, "hd15iqr": 0.1732538390024274, "ops": 5.939602347414667, "total": 1.1785300750052556, "data": [ 0.16951855400111526, 0.17012289500053157, 0.1732538390024274, 0.1616739819983195, 0.16210973700071918, 0.16948769800001173, 0.17236337000213098 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[3-4]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-4]", "params": { "zoom": 3, "item_width": 4 }, "param": "3-4", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.11248960100056138, "max": 0.12265536899940344, "mean": 0.11735441244430452, "stddev": 0.0035863367674561345, "rounds": 9, "median": 0.11760097699880134, "iqr": 0.00569761775295774, "q1": 0.1145561129978887, "q3": 0.12025373075084644, "iqr_outliers": 0, "stddev_outliers": 3, "outliers": "3;0", "ld15iqr": 0.11248960100056138, "hd15iqr": 0.12265536899940344, "ops": 8.52119642688844, "total": 1.0561897119987407, "data": [ 0.11948643000141601, 0.11310462300025392, 0.12265536899940344, 0.11510762000034447, 0.12055026199959684, 0.1150399429971003, 0.11760097699880134, 0.12015488700126298, 0.11248960100056138 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[3-5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-5]", "params": { "zoom": 3, "item_width": 5 }, "param": "3-5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.08600778799882391, "max": 0.11819229899992933, "mean": 0.09380789238415534, "stddev": 0.008470383289679074, "rounds": 13, "median": 0.09278260499922908, "iqr": 0.009015565500703815, "q1": 0.08737023674893862, "q3": 0.09638580224964244, "iqr_outliers": 1, "stddev_outliers": 1, "outliers": "1;1", "ld15iqr": 0.08600778799882391, "hd15iqr": 0.11819229899992933, "ops": 10.66008386485086, "total": 1.2195026009940193, "data": [ 0.0995918389999133, 0.09643121099725249, 0.09386423999967519, 0.09064780799963046, 0.08600778799882391, 0.09290014300131588, 0.09637066600043909, 0.08746462799899746, 0.11819229899992933, 0.09278260499922908, 0.08708706299876212, 0.09196810800131061, 0.08619420299874037 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[3-6]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-6]", "params": { "zoom": 3, "item_width": 6 }, "param": "3-6", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.06886182499874849, "max": 0.08020066499739187, "mean": 0.0725205675997131, "stddev": 0.003679136903194388, "rounds": 15, "median": 0.07074371199996676, "iqr": 0.005597590749857773, "q1": 0.06958281574952707, "q3": 0.07518040649938484, "iqr_outliers": 0, "stddev_outliers": 2, "outliers": "2;0", "ld15iqr": 0.06886182499874849, "hd15iqr": 0.08020066499739187, "ops": 13.789191578306898, "total": 1.0878085139956966, "data": [ 0.07045519399980549, 0.07074371199996676, 0.06936405300075421, 0.06957183299891767, 0.07550127799913753, 0.078494495999621, 0.08020066499739187, 0.07361344899982214, 0.07421779200012679, 0.07616023700029473, 0.06961576400135527, 0.06898055299825501, 0.07236440099950414, 0.06966326200199546, 0.06886182499874849 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[3-8]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-8]", "params": { "zoom": 3, "item_width": 8 }, "param": "3-8", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.05374020599992946, "max": 0.05982637700071791, "mean": 0.0563469560555758, "stddev": 0.0016293739578158342, "rounds": 18, "median": 0.05618607450014679, "iqr": 0.0021514159998332616, "q1": 0.05518398200001684, "q3": 0.0573353979998501, "iqr_outliers": 0, "stddev_outliers": 4, "outliers": "4;0", "ld15iqr": 0.05374020599992946, "hd15iqr": 0.05982637700071791, "ops": 17.747187603420596, "total": 1.0142452090003644, "data": [ 0.055682651000097394, 0.05374020599992946, 0.05650308599797427, 0.059472557997651165, 0.05982637700071791, 0.057392383001570124, 0.056449657000484876, 0.055928424997546244, 0.05518398200001684, 0.05479692000153591, 0.0573353979998501, 0.057961112001066795, 0.055098496999562485, 0.05566840800020145, 0.05707300100038992, 0.05644372400274733, 0.055237412998394575, 0.05445141100062756 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[3-10]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[3-10]", "params": { "zoom": 3, "item_width": 10 }, "param": "3-10", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.047568563000822905, "max": 0.05683792700074264, "mean": 0.05166188815055648, "stddev": 0.0025601894529204182, "rounds": 20, "median": 0.05139943300127925, "iqr": 0.002679176501260372, "q1": 0.05012959849955223, "q3": 0.0528087750008126, "iqr_outliers": 1, "stddev_outliers": 6, "outliers": "6;1", "ld15iqr": 0.047568563000822905, "hd15iqr": 0.05683792700074264, "ops": 19.356628954128315, "total": 1.0332377630111296, "data": [ 0.05180252200079849, 0.048698882001190213, 0.049534752000909066, 0.050387244002195075, 0.05197231200145325, 0.05162442900109454, 0.056070921000355156, 0.047568563000822905, 0.05047629700129619, 0.05343033399913111, 0.04820377700161771, 0.05592725600217818, 0.05117443700146396, 0.05683792700074264, 0.049871952996909386, 0.05052141399937682, 0.051092513000185136, 0.0516481759987073, 0.05420683799820836, 0.0521872160024941 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-0.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-0.5]", "params": { "zoom": 4, "item_width": 0.5 }, "param": "4-0.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 1.1995517129980726, "max": 1.2396531550002692, "mean": 1.2173349196661245, "stddev": 0.020431746778374022, "rounds": 3, "median": 1.2127998910000315, "iqr": 0.030076081501647423, "q1": 1.2028637574985623, "q3": 1.2329398390002098, "iqr_outliers": 0, "stddev_outliers": 1, "outliers": "1;0", "ld15iqr": 1.1995517129980726, "hd15iqr": 1.2396531550002692, "ops": 0.8214666184670588, "total": 3.6520047589983733, "data": [ 1.2396531550002692, 1.1995517129980726, 1.2127998910000315 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-0.75]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-0.75]", "params": { "zoom": 4, "item_width": 0.75 }, "param": "4-0.75", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.5314996209999663, "max": 0.5415407899999991, "mean": 0.5360264716667492, "stddev": 0.0050928958379338005, "rounds": 3, "median": 0.5350390040002821, "iqr": 0.007530876750024618, "q1": 0.5323844667500452, "q3": 0.5399153435000699, "iqr_outliers": 0, "stddev_outliers": 1, "outliers": "1;0", "ld15iqr": 0.5314996209999663, "hd15iqr": 0.5415407899999991, "ops": 1.8655795055989435, "total": 1.6080794150002475, "data": [ 0.5350390040002821, 0.5314996209999663, 0.5415407899999991 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-1]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-1]", "params": { "zoom": 4, "item_width": 1 }, "param": "4-1", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.326569004002522, "max": 0.3346178480023809, "mean": 0.32932656225148094, "stddev": 0.003615016240426719, "rounds": 4, "median": 0.3280596985005104, "iqr": 0.004467878499781364, "q1": 0.32709262300159025, "q3": 0.3315605015013716, "iqr_outliers": 0, "stddev_outliers": 1, "outliers": "1;0", "ld15iqr": 0.326569004002522, "hd15iqr": 0.3346178480023809, "ops": 3.036499677291072, "total": 1.3173062490059237, "data": [ 0.326569004002522, 0.3285031550003623, 0.32761624200065853, 0.3346178480023809 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-1.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-1.5]", "params": { "zoom": 4, "item_width": 1.5 }, "param": "4-1.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.17086795599971083, "max": 0.18105750900213025, "mean": 0.17587526450127675, "stddev": 0.004035728323823642, "rounds": 6, "median": 0.17464539750108088, "iqr": 0.006878126998344669, "q1": 0.17357860000265646, "q3": 0.18045672700100113, "iqr_outliers": 0, "stddev_outliers": 3, "outliers": "3;0", "ld15iqr": 0.17086795599971083, "hd15iqr": 0.18105750900213025, "ops": 5.685847881085863, "total": 1.0552515870076604, "data": [ 0.17497487199943862, 0.17431592300272314, 0.17086795599971083, 0.17357860000265646, 0.18105750900213025, 0.18045672700100113 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-2]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-2]", "params": { "zoom": 4, "item_width": 2 }, "param": "4-2", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.11330923400237225, "max": 0.14413199599948712, "mean": 0.1252486348890266, "stddev": 0.009915628744136859, "rounds": 9, "median": 0.1253533779999998, "iqr": 0.013842049500453868, "q1": 0.11569722474996524, "q3": 0.1295392742504191, "iqr_outliers": 0, "stddev_outliers": 3, "outliers": "3;0", "ld15iqr": 0.11330923400237225, "hd15iqr": 0.14413199599948712, "ops": 7.984118955755683, "total": 1.1272377140012395, "data": [ 0.1253533779999998, 0.11330923400237225, 0.12463980100073968, 0.1159379529999569, 0.11497503999999026, 0.12647421499787015, 0.12787050000042655, 0.14413199599948712, 0.1345455970003968 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-3]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-3]", "params": { "zoom": 4, "item_width": 3 }, "param": "4-3", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.06794780299969716, "max": 0.07971649299724959, "mean": 0.07236937333267027, "stddev": 0.0032234028131627192, "rounds": 15, "median": 0.07156555899928208, "iqr": 0.004010758501863165, "q1": 0.0702488237475336, "q3": 0.07425958224939677, "iqr_outliers": 0, "stddev_outliers": 4, "outliers": "4;0", "ld15iqr": 0.06794780299969716, "hd15iqr": 0.07971649299724959, "ops": 13.817999990177643, "total": 1.085540599990054, "data": [ 0.07195737400252256, 0.07060383200223441, 0.07430707499952405, 0.06810927799961064, 0.07010990799972205, 0.07129841300047701, 0.07156555899928208, 0.07971649299724959, 0.07411710399901494, 0.07689898899843683, 0.07484374299747287, 0.06794780299969716, 0.07022745199719793, 0.07031293899854063, 0.07352463799907127 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-4]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-4]", "params": { "zoom": 4, "item_width": 4 }, "param": "4-4", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.055022680997353746, "max": 0.07121412800188409, "mean": 0.0596297509411456, "stddev": 0.004634585881486992, "rounds": 17, "median": 0.05803015600031358, "iqr": 0.0026907579995167907, "q1": 0.05688350125001307, "q3": 0.05957425924952986, "iqr_outliers": 3, "stddev_outliers": 3, "outliers": "3;3", "ld15iqr": 0.055022680997353746, "hd15iqr": 0.06487147699954221, "ops": 16.77015221792553, "total": 1.0137057659994753, "data": [ 0.05694494400086114, 0.05621355599942035, 0.06983209000100032, 0.07121412800188409, 0.06487147699954221, 0.059164040998439305, 0.060804914002801524, 0.056618432001414476, 0.05837447899830295, 0.055022680997353746, 0.057981475998531096, 0.05858107200037921, 0.0578116910000972, 0.05820825400223839, 0.05803015600031358, 0.05669917299746885, 0.05733320199942682 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-5]", "params": { "zoom": 4, "item_width": 5 }, "param": "4-5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.048144559001229936, "max": 0.05387930500000948, "mean": 0.05032130766700166, "stddev": 0.0013144622688698053, "rounds": 21, "median": 0.050033582003379706, "iqr": 0.0009133455005212454, "q1": 0.049734376998458174, "q3": 0.05064772249897942, "iqr_outliers": 3, "stddev_outliers": 3, "outliers": "3;3", "ld15iqr": 0.04908135299774585, "hd15iqr": 0.0535943450013292, "ops": 19.872297568605372, "total": 1.0567474610070349, "data": [ 0.0535943450013292, 0.05037315099980333, 0.049767618998885155, 0.05091931800052407, 0.05014399900028366, 0.050104821002605604, 0.05055837699910626, 0.049749811998481164, 0.04908135299774585, 0.049688071998389205, 0.050033582003379706, 0.0509157589985989, 0.050990560001082486, 0.04938530599974911, 0.049980152001808165, 0.04927488500106847, 0.04977712100298959, 0.048144559001229936, 0.04999440000028699, 0.050390964999678545, 0.05387930500000948 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-6]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-6]", "params": { "zoom": 4, "item_width": 6 }, "param": "4-6", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.043895156999496976, "max": 0.04860524800096755, "mean": 0.046354956408487684, "stddev": 0.0013497296070651744, "rounds": 22, "median": 0.04642711300039082, "iqr": 0.0015482629969483241, "q1": 0.04566782300025807, "q3": 0.04721608599720639, "iqr_outliers": 0, "stddev_outliers": 8, "outliers": "8;0", "ld15iqr": 0.043895156999496976, "hd15iqr": 0.04860524800096755, "ops": 21.572666171613484, "total": 1.019809040986729, "data": [ 0.04666873200039845, 0.04600739599845838, 0.045710566999332514, 0.04721608599720639, 0.043895156999496976, 0.04667348099974333, 0.04566782300025807, 0.04652031700243242, 0.04593021999971825, 0.04669129100147984, 0.04403644800186157, 0.04860524800096755, 0.04782636699746945, 0.04569038199770148, 0.04524751200005994, 0.04787148499963223, 0.04827398699853802, 0.046333908998349216, 0.04409462699914002, 0.04688363699824549, 0.04830604399830918, 0.04565832499793032 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-8]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-8]", "params": { "zoom": 4, "item_width": 8 }, "param": "4-8", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.03777335199993104, "max": 0.0432480769995891, "mean": 0.04046258422170434, "stddev": 0.0014892429420970946, "rounds": 27, "median": 0.04061341799751972, "iqr": 0.0017895870014399407, "q1": 0.03936346749924269, "q3": 0.04115305450068263, "iqr_outliers": 0, "stddev_outliers": 9, "outliers": "9;0", "ld15iqr": 0.03777335199993104, "hd15iqr": 0.0432480769995891, "ops": 24.714190139729013, "total": 1.092489773986017, "data": [ 0.039341798998066224, 0.0414837219977926, 0.038832438996905694, 0.0399212099982833, 0.03942847300277208, 0.04115958400143427, 0.04072858699873905, 0.0432480769995891, 0.041842292001092574, 0.03887399599989294, 0.03982503799852566, 0.03982266300226911, 0.039465280999138486, 0.04093755499707186, 0.039434409998648334, 0.04061341799751972, 0.038854998998431256, 0.04075708299933467, 0.04294649800067418, 0.04074164800113067, 0.03777335199993104, 0.039088900997739984, 0.04110259200024302, 0.038892993001354625, 0.043079480001324555, 0.04113346599842771, 0.04316021799968439 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[4-10]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[4-10]", "params": { "zoom": 4, "item_width": 10 }, "param": "4-10", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.03276643600111129, "max": 0.03718326400121441, "mean": 0.03496493973090345, "stddev": 0.001112401201614798, "rounds": 26, "median": 0.03509476799990807, "iqr": 0.0016206899999815505, "q1": 0.03410217000055127, "q3": 0.03572286000053282, "iqr_outliers": 0, "stddev_outliers": 8, "outliers": "8;0", "ld15iqr": 0.03276643600111129, "hd15iqr": 0.03718326400121441, "ops": 28.600077897922386, "total": 0.9090884330034896, "data": [ 0.03396681500089471, 0.03627852499994333, 0.035353603001567535, 0.03601612800048315, 0.03626427700146451, 0.03524199599996791, 0.03479556399906869, 0.034837120001611765, 0.03572286000053282, 0.033217616997717414, 0.03414016399983666, 0.03410217000055127, 0.03650292800011812, 0.03567655399820069, 0.03583802900175215, 0.03504727499966975, 0.03346695399886812, 0.03514226100014639, 0.03276643600111129, 0.03558394300125656, 0.03486442800203804, 0.03718326400121441, 0.03358331300114514, 0.03385520899973926, 0.03416866199768265, 0.0354723379969073 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-0.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-0.5]", "params": { "zoom": 5, "item_width": 0.5 }, "param": "5-0.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.3528902499965625, "max": 0.3613380259994301, "mean": 0.3563750256653293, "stddev": 0.00441362560369112, "rounds": 3, "median": 0.3548968009999953, "iqr": 0.00633583200215071, "q1": 0.3533918877474207, "q3": 0.3597277197495714, "iqr_outliers": 0, "stddev_outliers": 1, "outliers": "1;0", "ld15iqr": 0.3528902499965625, "hd15iqr": 0.3613380259994301, "ops": 2.8060327688032127, "total": 1.069125076995988, "data": [ 0.3548968009999953, 0.3613380259994301, 0.3528902499965625 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-0.75]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-0.75]", "params": { "zoom": 5, "item_width": 0.75 }, "param": "5-0.75", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.1738603650010191, "max": 0.18968734400186804, "mean": 0.1816396906663916, "stddev": 0.006070635494990262, "rounds": 6, "median": 0.1798818664992723, "iqr": 0.00968734199705068, "q1": 0.17841967999993358, "q3": 0.18810702199698426, "iqr_outliers": 0, "stddev_outliers": 3, "outliers": "3;0", "ld15iqr": 0.1738603650010191, "hd15iqr": 0.18968734400186804, "ops": 5.50540466310664, "total": 1.0898381439983496, "data": [ 0.1738603650010191, 0.17841967999993358, 0.18043218900129432, 0.17933154399725026, 0.18810702199698426, 0.18968734400186804 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-1]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-1]", "params": { "zoom": 5, "item_width": 1 }, "param": "5-1", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.11908579000009922, "max": 0.14131123699917225, "mean": 0.13000069355580812, "stddev": 0.009213577351100307, "rounds": 9, "median": 0.1281640450033592, "iqr": 0.019572380000681733, "q1": 0.12101755949879589, "q3": 0.14058993949947762, "iqr_outliers": 0, "stddev_outliers": 5, "outliers": "5;0", "ld15iqr": 0.11908579000009922, "hd15iqr": 0.14131123699917225, "ops": 7.69226665372142, "total": 1.1700062420022732, "data": [ 0.12073260299803223, 0.12111254499905044, 0.11908579000009922, 0.1252064270011033, 0.14116875800027628, 0.1281640450033592, 0.13282783700196887, 0.14131123699917225, 0.1403969999992114 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-1.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-1.5]", "params": { "zoom": 5, "item_width": 1.5 }, "param": "5-1.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.07683741799701238, "max": 0.08334037099848501, "mean": 0.07977976357064367, "stddev": 0.001969583375656267, "rounds": 14, "median": 0.0792417389984621, "iqr": 0.003077533001487609, "q1": 0.0782598229998257, "q3": 0.08133735600131331, "iqr_outliers": 0, "stddev_outliers": 3, "outliers": "3;0", "ld15iqr": 0.07683741799701238, "hd15iqr": 0.08334037099848501, "ops": 12.534506938147496, "total": 1.1169166899890115, "data": [ 0.07964304999768501, 0.07898171299893875, 0.0782598229998257, 0.08133735600131331, 0.0809538520006754, 0.07811615699756658, 0.08290818000023137, 0.08173985699977493, 0.07891759799895226, 0.0780294860014692, 0.07683741799701238, 0.07950176499798545, 0.07835006399909616, 0.08334037099848501 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-2]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-2]", "params": { "zoom": 5, "item_width": 2 }, "param": "5-2", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.05488625500220223, "max": 0.0652931129989156, "mean": 0.05818601472209492, "stddev": 0.0029684394659227147, "rounds": 18, "median": 0.057458582497929456, "iqr": 0.0033470590024080593, "q1": 0.05617568099842174, "q3": 0.0595227400008298, "iqr_outliers": 2, "stddev_outliers": 4, "outliers": "4;2", "ld15iqr": 0.05488625500220223, "hd15iqr": 0.06502834500133758, "ops": 17.18626038879186, "total": 1.0473482649977086, "data": [ 0.05642501900001662, 0.0652931129989156, 0.05617568099842174, 0.05500854599813465, 0.05781774499700987, 0.05565800999829662, 0.05653662999975495, 0.05488625500220223, 0.05593465900165029, 0.05695694200039725, 0.05792698200093582, 0.0595227400008298, 0.05911192699932144, 0.05862750099913683, 0.059550049001700245, 0.06502834500133758, 0.05978870100079803, 0.057099419998849044 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-3]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-3]", "params": { "zoom": 5, "item_width": 3 }, "param": "5-3", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.04489852500046254, "max": 0.05440302599890856, "mean": 0.048053970475891786, "stddev": 0.002020601273945267, "rounds": 21, "median": 0.047797962000913685, "iqr": 0.0021989167489664396, "q1": 0.04697752375068376, "q3": 0.0491764404996502, "iqr_outliers": 1, "stddev_outliers": 5, "outliers": "5;1", "ld15iqr": 0.04489852500046254, "hd15iqr": 0.05440302599890856, "ops": 20.80993495639846, "total": 1.0091333799937274, "data": [ 0.04763054699651548, 0.04489852500046254, 0.047797962000913685, 0.04580801299744053, 0.04957300499881967, 0.04843555300249136, 0.04801880300146877, 0.049190688998351106, 0.04728741300277761, 0.049315358002786525, 0.048169592999329325, 0.047464323997701285, 0.05080188200008706, 0.046753120001085335, 0.0470523250005499, 0.04726841699812212, 0.04648834699764848, 0.05440302599890856, 0.04917169100008323, 0.04577358000096865, 0.047831206997216213 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-4]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-4]", "params": { "zoom": 5, "item_width": 4 }, "param": "5-4", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.03521118299977388, "max": 0.042654497003240976, "mean": 0.038113435110938516, "stddev": 0.0017210758691116373, "rounds": 27, "median": 0.037680810997699155, "iqr": 0.0016856995016496512, "q1": 0.03702540874837723, "q3": 0.03871110825002688, "iqr_outliers": 2, "stddev_outliers": 8, "outliers": "8;2", "ld15iqr": 0.03521118299977388, "hd15iqr": 0.04135794299872941, "ops": 26.237467105477485, "total": 1.02906274799534, "data": [ 0.03717857399897184, 0.04093288200238021, 0.037019471998064546, 0.039954528998350725, 0.037556142000539694, 0.04005070199855254, 0.03872921500078519, 0.042654497003240976, 0.038476314999570604, 0.03755851599999005, 0.03729255600046599, 0.03865678799775196, 0.037043218999315286, 0.03655166799944709, 0.03736498299986124, 0.03565524099758477, 0.03813792900109547, 0.03521118299977388, 0.03823172700140276, 0.03802275899943197, 0.03685324800244416, 0.03694704699955764, 0.037680810997699155, 0.04135794299872941, 0.03627146300277673, 0.03928250799799571, 0.03839083099956042 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-5]", "params": { "zoom": 5, "item_width": 5 }, "param": "5-5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.03793965599834337, "max": 0.04555513000013889, "mean": 0.04055096839983889, "stddev": 0.001934729870115932, "rounds": 30, "median": 0.04005605349993857, "iqr": 0.0014117259997874498, "q1": 0.03940837000118336, "q3": 0.04082009600097081, "iqr_outliers": 4, "stddev_outliers": 7, "outliers": "7;4", "ld15iqr": 0.03793965599834337, "hd15iqr": 0.04337045899956138, "ops": 24.66032352519534, "total": 1.2165290519951668, "data": [ 0.03940837000118336, 0.03993316499690991, 0.040693049999390496, 0.04137338500004262, 0.040091078997647855, 0.04555513000013889, 0.04147193300013896, 0.03979306199835264, 0.039633960001083324, 0.04014094599915552, 0.03979187299773912, 0.03993672700016759, 0.040509015998395626, 0.04054226000152994, 0.04337045899956138, 0.03941786699942895, 0.040662180002982495, 0.038028702001611236, 0.03920414999811328, 0.04003408700009459, 0.045365158999629784, 0.03905455000131042, 0.040078019999782555, 0.038662733997625764, 0.04082009600097081, 0.042519153001194354, 0.04443430100218393, 0.03793965599834337, 0.03873634800038417, 0.039327634000073886 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-6]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-6]", "params": { "zoom": 5, "item_width": 6 }, "param": "5-6", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.032226272000116296, "max": 0.039702834001218434, "mean": 0.03612378748385386, "stddev": 0.0015639443296231721, "rounds": 31, "median": 0.03627503800089471, "iqr": 0.0017682202478681575, "q1": 0.03527353050230886, "q3": 0.037041750750177016, "iqr_outliers": 2, "stddev_outliers": 8, "outliers": "8;2", "ld15iqr": 0.03333166900119977, "hd15iqr": 0.039702834001218434, "ops": 27.682590050862384, "total": 1.1198374119994696, "data": [ 0.03657423999902676, 0.03635696099809138, 0.03394788699733908, 0.03865205299734953, 0.035572141998272855, 0.03539641799943638, 0.03511977200105321, 0.037492039999051485, 0.03589865500180167, 0.03558995099956519, 0.035221881000325084, 0.03523375500299153, 0.03443706300095073, 0.03573005499856663, 0.03539285700026085, 0.037199960999714676, 0.03654574699976365, 0.032226272000116296, 0.03487518600013573, 0.03635221399963484, 0.0382186829992861, 0.03577517599842395, 0.038712609002686804, 0.03627503800089471, 0.03629640900180675, 0.03712872200048878, 0.03653862300052424, 0.03333166900119977, 0.03698030699888477, 0.037062232000607764, 0.039702834001218434 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-8]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-8]", "params": { "zoom": 5, "item_width": 8 }, "param": "5-8", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.030467855001916178, "max": 0.03646382800070569, "mean": 0.03345623115592389, "stddev": 0.0012971261642753505, "rounds": 32, "median": 0.033450998498665285, "iqr": 0.0016723429998819483, "q1": 0.03265430600004038, "q3": 0.03432664899992233, "iqr_outliers": 0, "stddev_outliers": 8, "outliers": "8;0", "ld15iqr": 0.030467855001916178, "hd15iqr": 0.03646382800070569, "ops": 29.889798266262165, "total": 1.0705993969895644, "data": [ 0.03300872199906735, 0.033955015998799354, 0.03407256100035738, 0.03350977099762531, 0.03443825599970296, 0.033723488999385154, 0.033799476997955935, 0.03339222599970526, 0.03351808200022788, 0.032727325000450946, 0.03438245200231904, 0.03174541200132808, 0.03243880700028967, 0.03119449600126245, 0.03338629100107937, 0.03163974199924269, 0.030467855001916178, 0.03351452199785854, 0.03527769399806857, 0.03313814099965384, 0.03427084599752561, 0.03279144299813197, 0.03513284099972225, 0.03254091800044989, 0.03258128699962981, 0.03308114999890677, 0.03646382800070569, 0.03286386999752722, 0.034547491999546764, 0.03583811000135029, 0.032556353999098064, 0.034600921000674134 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[5-10]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[5-10]", "params": { "zoom": 5, "item_width": 10 }, "param": "5-10", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.03186058700157446, "max": 0.039468946000852156, "mean": 0.036132189964389126, "stddev": 0.0015354948988810695, "rounds": 28, "median": 0.0361070445014775, "iqr": 0.0010786809998535318, "q1": 0.035639239500596887, "q3": 0.03671792050045042, "iqr_outliers": 4, "stddev_outliers": 7, "outliers": "7;4", "ld15iqr": 0.03407731799961766, "hd15iqr": 0.03914836900003138, "ops": 27.676152510699513, "total": 1.0117013190028956, "data": [ 0.03186058700157446, 0.03628336000110721, 0.03606133100038278, 0.03678203500021482, 0.034916751999844564, 0.03575975299827405, 0.03663480899922433, 0.036408030002348823, 0.03615038200223353, 0.03407731799961766, 0.03781619399887859, 0.03565883000192116, 0.039468946000852156, 0.03572532099860837, 0.03914836900003138, 0.03829468500043731, 0.03606370700072148, 0.03556978099732078, 0.03700169099829509, 0.0358013100012613, 0.03665380600068602, 0.036865149002551334, 0.03629761000047438, 0.03508416700060479, 0.03599721799764666, 0.036259615000744816, 0.03561964899927261, 0.033440913997765165 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-0.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-0.5]", "params": { "zoom": 6, "item_width": 0.5 }, "param": "6-0.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.1268487230008759, "max": 0.14295829899856471, "mean": 0.1350773039998785, "stddev": 0.005595496365896534, "rounds": 8, "median": 0.1347099775011884, "iqr": 0.009163152501059812, "q1": 0.13076628749877273, "q3": 0.13992943999983254, "iqr_outliers": 0, "stddev_outliers": 3, "outliers": "3;0", "ld15iqr": 0.1268487230008759, "hd15iqr": 0.14295829899856471, "ops": 7.403168188794318, "total": 1.080618431999028, "data": [ 0.14295829899856471, 0.13436564999938128, 0.13881217000016477, 0.1410467099995003, 0.13178797499858774, 0.1297445999989577, 0.13505430500299553, 0.1268487230008759 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-0.75]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-0.75]", "params": { "zoom": 6, "item_width": 0.75 }, "param": "6-0.75", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.07463744200140354, "max": 0.08529722000093898, "mean": 0.07939257461503775, "stddev": 0.002691959172431208, "rounds": 13, "median": 0.0788595570011239, "iqr": 0.0035290212490508566, "q1": 0.07768084224881022, "q3": 0.08120986349786108, "iqr_outliers": 0, "stddev_outliers": 2, "outliers": "2;0", "ld15iqr": 0.07463744200140354, "hd15iqr": 0.08529722000093898, "ops": 12.595636365854672, "total": 1.0321034699954907, "data": [ 0.07994239300023764, 0.07783370899778674, 0.0776900429991656, 0.0788595570011239, 0.07463744200140354, 0.08125438799834228, 0.0776532399977441, 0.08181836599760572, 0.08119502199770068, 0.07718187399950693, 0.08529722000093898, 0.0806061100010993, 0.07813410600283532 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-1]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-1]", "params": { "zoom": 6, "item_width": 1 }, "param": "6-1", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.05851480500132311, "max": 0.06430774700129405, "mean": 0.06168284427783672, "stddev": 0.0016228951986026318, "rounds": 18, "median": 0.06135784850084747, "iqr": 0.0027047200019296724, "q1": 0.060831270999187836, "q3": 0.06353599100111751, "iqr_outliers": 0, "stddev_outliers": 7, "outliers": "7;0", "ld15iqr": 0.05851480500132311, "hd15iqr": 0.06430774700129405, "ops": 16.211963175623378, "total": 1.110291197001061, "data": [ 0.06118627600153559, 0.06430774700129405, 0.06387675199948717, 0.06353599100111751, 0.06052612900020904, 0.060831270999187836, 0.062220438001531875, 0.05851480500132311, 0.06128720299966517, 0.06124683300004108, 0.05954658799964818, 0.06385656800193829, 0.061428494002029765, 0.06006544800038682, 0.06371408899940434, 0.06163390099754906, 0.06156741099766805, 0.06094525299704401 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-1.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-1.5]", "params": { "zoom": 6, "item_width": 1.5 }, "param": "6-1.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.042510893999860855, "max": 0.049666887000057613, "mean": 0.04612569804472018, "stddev": 0.0016540591589389979, "rounds": 22, "median": 0.04573744499793975, "iqr": 0.0020493189986154903, "q1": 0.04529991500021424, "q3": 0.04734923399882973, "iqr_outliers": 0, "stddev_outliers": 8, "outliers": "8;0", "ld15iqr": 0.042510893999860855, "hd15iqr": 0.049666887000057613, "ops": 21.679888703916664, "total": 1.014765356983844, "data": [ 0.04734923399882973, 0.04429187900313991, 0.042510893999860855, 0.04473474900078145, 0.04529991500021424, 0.04554212900256971, 0.04442604599898914, 0.045679858998482814, 0.047726801996759605, 0.04783722299907822, 0.04555756400077371, 0.04424794700025814, 0.048447506997035816, 0.04612629199982621, 0.04820410599859315, 0.046539479997591116, 0.04555162799806567, 0.04567985799803864, 0.046430245998635655, 0.049666887000057613, 0.04712008099886589, 0.04579503099739668 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-2]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-2]", "params": { "zoom": 6, "item_width": 2 }, "param": "6-2", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.042301935001887614, "max": 0.0495885320015077, "mean": 0.044751182250062506, "stddev": 0.0019226710688970588, "rounds": 24, "median": 0.04421233649918577, "iqr": 0.0012057274998369394, "q1": 0.04363173349884164, "q3": 0.04483746099867858, "iqr_outliers": 4, "stddev_outliers": 6, "outliers": "6;4", "ld15iqr": 0.042301935001887614, "hd15iqr": 0.04736823900020681, "ops": 22.345778362058876, "total": 1.0740283740015002, "data": [ 0.04329690899976413, 0.04402592499900493, 0.04481430699888733, 0.0441862139996374, 0.043782525001006434, 0.044828555997810327, 0.04367328999796882, 0.044974595999519806, 0.043363400000089314, 0.04945198999848799, 0.04400217899819836, 0.04736823900020681, 0.0480746960020042, 0.0495885320015077, 0.04359017699971446, 0.04392381500292686, 0.04484636599954683, 0.0427697380000609, 0.042301935001887614, 0.044630275002418784, 0.044303760998445796, 0.04423845899873413, 0.04352725300122984, 0.04446523700244143 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-3]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-3]", "params": { "zoom": 6, "item_width": 3 }, "param": "6-3", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.03260270100145135, "max": 0.04594939700109535, "mean": 0.03932017700019599, "stddev": 0.0033886416698431653, "rounds": 27, "median": 0.039124662998801796, "iqr": 0.0042791097512235865, "q1": 0.037114231248779106, "q3": 0.04139334100000269, "iqr_outliers": 0, "stddev_outliers": 8, "outliers": "8;0", "ld15iqr": 0.03260270100145135, "hd15iqr": 0.04594939700109535, "ops": 25.432235465140852, "total": 1.0616447790052916, "data": [ 0.03918521699961275, 0.039124662998801796, 0.04085459300040384, 0.03933006999795907, 0.038238919998548226, 0.04558013800124172, 0.045501774999138433, 0.0415183059994888, 0.03773312100020121, 0.04012320200126851, 0.03787322599964682, 0.04200154600039241, 0.04346076600268134, 0.04594939700109535, 0.0377901150022808, 0.042507349000516115, 0.04101844600154436, 0.040246685999591136, 0.035464149001199985, 0.037032602998806396, 0.037015980000433046, 0.03260270100145135, 0.037359115998697234, 0.034945289000461344, 0.03660397899875534, 0.03765594699871144, 0.034927479002362816 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-4]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-4]", "params": { "zoom": 6, "item_width": 4 }, "param": "6-4", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.036126680999586824, "max": 0.04487012900062837, "mean": 0.03896573932161118, "stddev": 0.002392502885028669, "rounds": 28, "median": 0.03843424250044336, "iqr": 0.0034972590001416393, "q1": 0.037080099999002414, "q3": 0.04057735899914405, "iqr_outliers": 0, "stddev_outliers": 8, "outliers": "8;0", "ld15iqr": 0.036126680999586824, "hd15iqr": 0.04487012900062837, "ops": 25.663570547098022, "total": 1.091040701005113, "data": [ 0.04196949600009248, 0.038218742000026396, 0.03735437100112904, 0.04305352200026391, 0.04069430999879842, 0.041581241002859315, 0.03641519799930393, 0.036954242998035625, 0.04046040799948969, 0.03879103300278075, 0.03729975600072066, 0.03675952399862581, 0.04079879699929734, 0.04363412499878905, 0.039502240000729216, 0.036126680999586824, 0.03678683200269006, 0.03678920699894661, 0.038649743000860326, 0.0372059569999692, 0.04487012900062837, 0.039962922000995604, 0.037445797002874315, 0.03743511100037722, 0.036178923000989016, 0.03801689899773919, 0.03899881599863875, 0.039086677999875974 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-5]", "params": { "zoom": 6, "item_width": 5 }, "param": "6-5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.03487881000182824, "max": 0.04174747499928344, "mean": 0.037700271428808003, "stddev": 0.002003219081389196, "rounds": 28, "median": 0.03710860299906926, "iqr": 0.0032580150000285357, "q1": 0.03605426000103762, "q3": 0.039312275001066155, "iqr_outliers": 0, "stddev_outliers": 10, "outliers": "10;0", "ld15iqr": 0.03487881000182824, "hd15iqr": 0.04174747499928344, "ops": 26.52500796680916, "total": 1.0556076000066241, "data": [ 0.036404519003554014, 0.03566244100147742, 0.04006503599885036, 0.041011334000359057, 0.037138285999390064, 0.035888034999516094, 0.03707891999874846, 0.03828286599673447, 0.035546086000977084, 0.035413106001215056, 0.036980372002290096, 0.03811782800039509, 0.03661348899913719, 0.03766901900235098, 0.03919829200094682, 0.03589159600232961, 0.03487881000182824, 0.03893351800070377, 0.03942625800118549, 0.04164892799963127, 0.04174747499928344, 0.034936987998662516, 0.036812959999224404, 0.039606730999366846, 0.03621692399974563, 0.03673459600031492, 0.03782693399989512, 0.03987625299851061 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-6]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-6]", "params": { "zoom": 6, "item_width": 6 }, "param": "6-6", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.030040474000998074, "max": 0.04595891800272511, "mean": 0.03633510730008614, "stddev": 0.0043045462862126484, "rounds": 30, "median": 0.036877674001516425, "iqr": 0.007704540999839082, "q1": 0.032137285998032894, "q3": 0.039841826997871976, "iqr_outliers": 0, "stddev_outliers": 10, "outliers": "10;0", "ld15iqr": 0.030040474000998074, "hd15iqr": 0.04595891800272511, "ops": 27.521592044331992, "total": 1.0900532190025842, "data": [ 0.03704330600157846, 0.04039037099937559, 0.04017309099799604, 0.03581323899925337, 0.039841826997871976, 0.04328981499929796, 0.037152539000089746, 0.041258303997892654, 0.039796708999347175, 0.037076551001518965, 0.035965216000477085, 0.04216067000015755, 0.037044493001303636, 0.03418779300045571, 0.04595891800272511, 0.040525724998587975, 0.03705755299961311, 0.03956755600302131, 0.03671204200145439, 0.03355851100059226, 0.03365587200096343, 0.03159586599940667, 0.031034261999593582, 0.03230351200181758, 0.032137285998032894, 0.030491656998492545, 0.03206011000293074, 0.030754054998396896, 0.03140589599934174, 0.030040474000998074 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-8]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-8]", "params": { "zoom": 6, "item_width": 8 }, "param": "6-8", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.029466998999851057, "max": 0.035118658997816965, "mean": 0.0317732569141039, "stddev": 0.0011116560823296769, "rounds": 35, "median": 0.03184639699975378, "iqr": 0.0011015400014002807, "q1": 0.03125985899987427, "q3": 0.03236139900127455, "iqr_outliers": 2, "stddev_outliers": 9, "outliers": "9;2", "ld15iqr": 0.02967003300000215, "hd15iqr": 0.035118658997816965, "ops": 31.473008974289563, "total": 1.1120639919936366, "data": [ 0.03141539699936402, 0.03188201500233845, 0.032318950001354096, 0.03188201599914464, 0.03234269700260484, 0.03293398499954492, 0.030158020999806467, 0.032220401997619774, 0.03297316599855549, 0.03193544500027201, 0.03320944299775874, 0.029466998999851057, 0.03263121700001648, 0.03162436600177898, 0.029878999997890787, 0.03160299400042277, 0.03261221899811062, 0.035118658997816965, 0.03143676900072023, 0.031933069996739505, 0.030959464998886688, 0.030906036001397297, 0.033446908000769326, 0.030955902999266982, 0.03253385599964531, 0.03125273499972536, 0.031281231000320986, 0.03155075399990892, 0.03188320400295197, 0.031553127999359276, 0.031422522999491775, 0.03236763300083112, 0.02967003300000215, 0.03184639699975378, 0.03085735599961481 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[6-10]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[6-10]", "params": { "zoom": 6, "item_width": 10 }, "param": "6-10", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.026975998996931594, "max": 0.04209300799993798, "mean": 0.03214526745681984, "stddev": 0.0029009541935060648, "rounds": 35, "median": 0.032535049998841714, "iqr": 0.003922621500350942, "q1": 0.029960930250126694, "q3": 0.033883551750477636, "iqr_outliers": 1, "stddev_outliers": 8, "outliers": "8;1", "ld15iqr": 0.026975998996931594, "hd15iqr": 0.04209300799993798, "ops": 31.10877834017969, "total": 1.1250843609886942, "data": [ 0.029685470999538666, 0.03109126299750642, 0.028255932997126365, 0.026975998996931594, 0.029716340999584645, 0.02902888099924894, 0.030137841000396293, 0.030155651998938993, 0.029178483000578126, 0.03001792199938791, 0.029941933000372956, 0.029191544999775942, 0.028652499000600073, 0.030491664001601748, 0.030851424002321437, 0.03384704099880764, 0.03787088199896971, 0.032096925999212544, 0.034644923001906136, 0.032717895999667235, 0.03391234399896348, 0.03392421700118575, 0.033438600999943446, 0.0338957220010343, 0.03235932600000524, 0.03323913299755077, 0.03395390199875692, 0.034150997998949606, 0.0335490249999566, 0.033031350998498965, 0.034173557000030996, 0.04209300799993798, 0.032535049998841714, 0.03348016000018106, 0.032797448002384044 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-0.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-0.5]", "params": { "zoom": 7, "item_width": 0.5 }, "param": "7-0.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.05863600000157021, "max": 0.06823670299854712, "mean": 0.06251938735273571, "stddev": 0.0022581559398568876, "rounds": 17, "median": 0.06236657000044943, "iqr": 0.002586880998023844, "q1": 0.06106378149979719, "q3": 0.06365066249782103, "iqr_outliers": 1, "stddev_outliers": 4, "outliers": "4;1", "ld15iqr": 0.05863600000157021, "hd15iqr": 0.06823670299854712, "ops": 15.995038376783489, "total": 1.062829584996507, "data": [ 0.06236657000044943, 0.06587986099839327, 0.0637236809998285, 0.06111394300023676, 0.06390177900175331, 0.061628056999325054, 0.06823670299854712, 0.06362632299715187, 0.062382007999985944, 0.06094534600197221, 0.06093109799985541, 0.05984113399972557, 0.0634387259997311, 0.06276551399787422, 0.06230958200103487, 0.05863600000157021, 0.06110325999907218 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-0.75]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-0.75]", "params": { "zoom": 7, "item_width": 0.75 }, "param": "7-0.75", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.04354155399778392, "max": 0.048738473997218534, "mean": 0.045375971043066114, "stddev": 0.0013899984103265528, "rounds": 23, "median": 0.04513969400068163, "iqr": 0.0020098417462577345, "q1": 0.04439405400171381, "q3": 0.046403895747971546, "iqr_outliers": 0, "stddev_outliers": 8, "outliers": "8;0", "ld15iqr": 0.04354155399778392, "hd15iqr": 0.048738473997218534, "ops": 22.038095869086856, "total": 1.0436473339905206, "data": [ 0.044486663999123266, 0.04644456099777017, 0.04354155399778392, 0.046033746999455616, 0.04451278499982436, 0.04513969400068163, 0.04362229300022591, 0.048738473997218534, 0.04672833400036325, 0.04391912500068429, 0.04572267000185093, 0.04478943299909588, 0.044901042001583846, 0.044947346999833826, 0.04373271399890655, 0.04566924100072356, 0.046489682001265464, 0.04436318400257733, 0.04370896799809998, 0.04515987799823051, 0.04768887899990659, 0.04702516499673948, 0.04628189999857568 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-1]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-1]", "params": { "zoom": 7, "item_width": 1 }, "param": "7-1", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.038548863001778955, "max": 0.044610154000110924, "mean": 0.04028780892568412, "stddev": 0.0012854161512075718, "rounds": 27, "median": 0.04029422899839119, "iqr": 0.0013024955014770967, "q1": 0.039389784499689995, "q3": 0.04069228000116709, "iqr_outliers": 1, "stddev_outliers": 6, "outliers": "6;1", "ld15iqr": 0.038548863001778955, "hd15iqr": 0.044610154000110924, "ops": 24.821404456236987, "total": 1.0877708409934712, "data": [ 0.03955571199912811, 0.04021467699931236, 0.03939067399915075, 0.0399344689976715, 0.03902379099963582, 0.0392315739991318, 0.03859041900068405, 0.04195054699812317, 0.040376155000558356, 0.042304368998884456, 0.04080359200088424, 0.039497534999100026, 0.04029422899839119, 0.03935624400037341, 0.04052694500205689, 0.039895288999105105, 0.038548863001778955, 0.04146493200096302, 0.04038209099962842, 0.03889556200010702, 0.041216779998649145, 0.0405756250002014, 0.044610154000110924, 0.04053881900108536, 0.04073116500148899, 0.03938948799986974, 0.040471140997397015 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-1.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-1.5]", "params": { "zoom": 7, "item_width": 1.5 }, "param": "7-1.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.03365115400083596, "max": 0.038333962998876814, "mean": 0.03571895903320789, "stddev": 0.0011727307440549206, "rounds": 30, "median": 0.03574915599892847, "iqr": 0.0013927309955761302, "q1": 0.03493702700143331, "q3": 0.03632975799700944, "iqr_outliers": 0, "stddev_outliers": 9, "outliers": "9;0", "ld15iqr": 0.03365115400083596, "hd15iqr": 0.038333962998876814, "ops": 27.996336597331986, "total": 1.0715687709962367, "data": [ 0.03577052699984051, 0.03538939600184676, 0.03459745000145631, 0.03440985300039756, 0.03530628500084276, 0.03578596399893286, 0.03455233400018187, 0.03680350099966745, 0.03682012299759663, 0.03631788499842514, 0.03365115400083596, 0.03578833799838321, 0.034114211000996875, 0.038068002002546564, 0.03616472100111423, 0.035592429998359876, 0.03632975799700944, 0.03572778499801643, 0.03622289999839268, 0.03500470399740152, 0.03406196899959468, 0.03412727199975052, 0.03548913299891865, 0.037474339002073975, 0.03661590400224668, 0.038333962998876814, 0.03522910899846465, 0.03493702700143331, 0.035793087001366075, 0.03708964700126671 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-2]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-2]", "params": { "zoom": 7, "item_width": 2 }, "param": "7-2", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.028906609000841854, "max": 0.035358532000827836, "mean": 0.0319112826878154, "stddev": 0.0015387649107082991, "rounds": 32, "median": 0.03188560699891241, "iqr": 0.0017311204992438434, "q1": 0.0308466970018344, "q3": 0.03257781750107824, "iqr_outliers": 1, "stddev_outliers": 9, "outliers": "9;1", "ld15iqr": 0.028906609000841854, "hd15iqr": 0.035358532000827836, "ops": 31.33687886453487, "total": 1.0211610460100928, "data": [ 0.03288058399994043, 0.0333008970010269, 0.03068165800141287, 0.0319900909998978, 0.03119695799978217, 0.028906609000841854, 0.030351583001902327, 0.03415102299913997, 0.03243890100202407, 0.0317550020008639, 0.03226198900301824, 0.029107267000654247, 0.03183811399867409, 0.03240684199772659, 0.031771623998793075, 0.030382453001948306, 0.03186779699899489, 0.031715819000964984, 0.030463191997114336, 0.031903416998829925, 0.030586674001824576, 0.03101173600225593, 0.02950977000000421, 0.03208745200026897, 0.03271673400013242, 0.03142492500046501, 0.035358532000827836, 0.0322239939996507, 0.03410471700044582, 0.03201383800114854, 0.03500708399951691, 0.03374377000000095 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-3]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-3]", "params": { "zoom": 7, "item_width": 3 }, "param": "7-3", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.029087087001244072, "max": 0.03676670400091098, "mean": 0.03163497772766277, "stddev": 0.0014185766524036627, "rounds": 33, "median": 0.03158284399978584, "iqr": 0.0012312565004322096, "q1": 0.030951482500313432, "q3": 0.03218273900074564, "iqr_outliers": 3, "stddev_outliers": 7, "outliers": "7;3", "ld15iqr": 0.0291345799996634, "hd15iqr": 0.03412371900049038, "ops": 31.610580181492075, "total": 1.0439542650128715, "data": [ 0.032865152999875136, 0.031371498000225984, 0.03136912400077563, 0.032038774999819, 0.030763588001718745, 0.031986534002498956, 0.029087087001244072, 0.03213851100008469, 0.03412371900049038, 0.03289365000091493, 0.030415704000915866, 0.03158284399978584, 0.031763317001605174, 0.03144511400023475, 0.03193191800164641, 0.030003701001987793, 0.029894466999394353, 0.031568596001307014, 0.0324092209993978, 0.03182387099877815, 0.03147717200045008, 0.030071378998400178, 0.030258975999458926, 0.0291345799996634, 0.03154959799940116, 0.032324921998224454, 0.03129788599835592, 0.03676670400091098, 0.03182624500186648, 0.03243890500016278, 0.03200196900070296, 0.032315423002728494, 0.031014113999844994 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-4]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-4]", "params": { "zoom": 7, "item_width": 4 }, "param": "7-4", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.027581562000705162, "max": 0.034331506001763046, "mean": 0.03041731352973062, "stddev": 0.001521611379613399, "rounds": 34, "median": 0.0303409059997648, "iqr": 0.0015363989987235982, "q1": 0.02964750799947069, "q3": 0.03118390699819429, "iqr_outliers": 1, "stddev_outliers": 12, "outliers": "12;1", "ld15iqr": 0.027581562000705162, "hd15iqr": 0.034331506001763046, "ops": 32.87601316344311, "total": 1.034188660010841, "data": [ 0.03198772500036284, 0.0319449810012884, 0.031002244002593216, 0.030888261000654893, 0.02975436599808745, 0.030262541000411147, 0.0298434169999382, 0.03118390699819429, 0.030980874002125347, 0.03254339400155004, 0.028778386000340106, 0.028194221999001456, 0.027581562000705162, 0.02964750799947069, 0.028952922999451403, 0.029886160002206452, 0.030547499998647254, 0.03085501799796475, 0.03128482899774099, 0.029976397003338207, 0.030267292000644375, 0.03216226299991831, 0.0294052940007532, 0.031698018003226025, 0.03062823800064507, 0.027803591998235788, 0.028349762000289047, 0.0302103009998973, 0.028399630002240883, 0.034331506001763046, 0.030406208999920636, 0.033328215999063104, 0.030275602999608964, 0.030826521000562934 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-5]", "params": { "zoom": 7, "item_width": 5 }, "param": "7-5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.029624953000165988, "max": 0.03544047200193745, "mean": 0.03163227453126183, "stddev": 0.0013722909373689459, "rounds": 32, "median": 0.0315466385018226, "iqr": 0.0019151540018356172, "q1": 0.030518415498590912, "q3": 0.03243356950042653, "iqr_outliers": 1, "stddev_outliers": 10, "outliers": "10;1", "ld15iqr": 0.029624953000165988, "hd15iqr": 0.03544047200193745, "ops": 31.613281523960946, "total": 1.0122327850003785, "data": [ 0.03236292300061905, 0.031078236999746878, 0.032562393997068284, 0.03205659299783292, 0.03353956299906713, 0.03250421600023401, 0.030679297000460792, 0.030446581997239264, 0.030127191999781644, 0.031597100001818035, 0.03289603400116903, 0.03085620999991079, 0.030072575002122903, 0.03132282899969141, 0.02995740400001523, 0.031661215998610714, 0.031255150999641046, 0.029624953000165988, 0.034652089001610875, 0.030336161002196604, 0.03097850300036953, 0.03012363000016194, 0.0320767800003523, 0.032229945001745364, 0.03059024899994256, 0.030080885997449514, 0.03328547599812737, 0.031496177001827164, 0.03174314100033371, 0.03544047200193745, 0.032525588998396415, 0.032073218000732595 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-6]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-6]", "params": { "zoom": 7, "item_width": 6 }, "param": "7-6", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.028555175998917548, "max": 0.03251253300186363, "mean": 0.030253447555676556, "stddev": 0.0010558645538548761, "rounds": 36, "median": 0.03013194450068113, "iqr": 0.001688376998572494, "q1": 0.029296067001268966, "q3": 0.03098444399984146, "iqr_outliers": 0, "stddev_outliers": 14, "outliers": "14;0", "ld15iqr": 0.028555175998917548, "hd15iqr": 0.03251253300186363, "ops": 33.05408410594073, "total": 1.089124112004356, "data": [ 0.030247113998484565, 0.030636556999525055, 0.029297254000994144, 0.029294880001543788, 0.02926163400115911, 0.03138338500139071, 0.029210578999482095, 0.029475352002918953, 0.030191310001100646, 0.02983748699989519, 0.030895395000698045, 0.02914765300010913, 0.030963072000304237, 0.03199960799975088, 0.02985529900252004, 0.029063352001685416, 0.03001083799972548, 0.031768078999448335, 0.030256614001700655, 0.029065726997941965, 0.032276254998578224, 0.030529698997270316, 0.030372971999895526, 0.028945805999683216, 0.02973894000024302, 0.02881163900019601, 0.03144750199862756, 0.03156029700039653, 0.028555175998917548, 0.02942786100174999, 0.030723233001481276, 0.030072579000261612, 0.03251253300186363, 0.031005815999378683, 0.029858859998057596, 0.03142375499737682 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-8]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-8]", "params": { "zoom": 7, "item_width": 8 }, "param": "7-8", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.028266658999200445, "max": 0.03293759799998952, "mean": 0.030398328735078823, "stddev": 0.0010968732816062303, "rounds": 34, "median": 0.030405034001887543, "iqr": 0.0014794090020586737, "q1": 0.02978168599656783, "q3": 0.031261094998626504, "iqr_outliers": 0, "stddev_outliers": 11, "outliers": "11;0", "ld15iqr": 0.028266658999200445, "hd15iqr": 0.03293759799998952, "ops": 32.89654535665403, "total": 1.03354317699268, "data": [ 0.03293759799998952, 0.031353705999208614, 0.028828263999457704, 0.028266658999200445, 0.02978168599656783, 0.030090391999692656, 0.029738943001575535, 0.02993010300269816, 0.030371787997864885, 0.03128009200008819, 0.03140357500160462, 0.030444216001342284, 0.030405034001887543, 0.031261094998626504, 0.030045274997974047, 0.03119460499874549, 0.03161017000093125, 0.03018062899718643, 0.029716384997300338, 0.02863117000015336, 0.02990754400161677, 0.030071395998675143, 0.031597108998539625, 0.030409783001232427, 0.029296071999851847, 0.030473897997580934, 0.028559929996845312, 0.030412158001126954, 0.031017694000183837, 0.030405034001887543, 0.03155436499946518, 0.028773649002687307, 0.032555280999076786, 0.031037879001814872 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[7-10]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[7-10]", "params": { "zoom": 7, "item_width": 10 }, "param": "7-10", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.027302555998176103, "max": 0.03296134999982314, "mean": 0.029292249388870713, "stddev": 0.0012300904623758812, "rounds": 36, "median": 0.02918980949834804, "iqr": 0.0015280890020221705, "q1": 0.02840973649836087, "q3": 0.029937825500383042, "iqr_outliers": 1, "stddev_outliers": 12, "outliers": "12;1", "ld15iqr": 0.027302555998176103, "hd15iqr": 0.03296134999982314, "ops": 34.1387234119323, "total": 1.0545209779993456, "data": [ 0.028638296000281116, 0.02874396799961687, 0.028501754000899382, 0.031631544003175804, 0.028944625999429263, 0.031008198002382414, 0.02980424899942591, 0.027741865000280086, 0.02796508199753589, 0.02837114799694973, 0.03085622000071453, 0.027866535001521697, 0.0292758909999975, 0.029145284999685828, 0.027721681999537395, 0.030627068001194857, 0.0274438470005407, 0.028330780001851963, 0.029078794999804813, 0.029919422002421925, 0.030410973999096313, 0.027302555998176103, 0.02914409799996065, 0.030698307000420755, 0.02995622899834416, 0.029785253998852568, 0.03296134999982314, 0.028354526999464724, 0.02957391000018106, 0.02928301499923691, 0.02968908000184456, 0.02860624000095413, 0.030064275000768248, 0.028448324999772012, 0.029392248998192372, 0.029234333997010253 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-0.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-0.5]", "params": { "zoom": 8, "item_width": 0.5 }, "param": "8-0.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.03788044600150897, "max": 0.044045042002835544, "mean": 0.04064544235996436, "stddev": 0.0015116283612815955, "rounds": 25, "median": 0.04059704699830036, "iqr": 0.0015732069959994988, "q1": 0.039664106003328925, "q3": 0.041237312999328424, "iqr_outliers": 1, "stddev_outliers": 9, "outliers": "9;1", "ld15iqr": 0.03788044600150897, "hd15iqr": 0.044045042002835544, "ops": 24.603004468343464, "total": 1.016136058999109, "data": [ 0.04063860200039926, 0.04158252599881962, 0.043160482000530465, 0.043389636997744674, 0.04110878299979959, 0.04084163500010618, 0.04137711899966234, 0.04247183400002541, 0.04102685999896494, 0.04059704699830036, 0.044045042002835544, 0.03788044600150897, 0.03829600999961258, 0.04054717999679269, 0.0391271369990136, 0.03996182800256065, 0.03905946099985158, 0.03968874300335301, 0.039131886998802656, 0.03985259399996721, 0.04119071099921712, 0.03959019500325667, 0.041112347000307636, 0.04018504600026063, 0.0402729069974157 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-0.75]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-0.75]", "params": { "zoom": 8, "item_width": 0.75 }, "param": "8-0.75", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.03196281700002146, "max": 0.03790182000011555, "mean": 0.03527197655182982, "stddev": 0.0015415483599651623, "rounds": 29, "median": 0.03531463800027268, "iqr": 0.002420364248791884, "q1": 0.034211019249596575, "q3": 0.03663138349838846, "iqr_outliers": 0, "stddev_outliers": 9, "outliers": "9;0", "ld15iqr": 0.03196281700002146, "hd15iqr": 0.03790182000011555, "ops": 28.35111886997789, "total": 1.0228873200030648, "data": [ 0.037126497001736425, 0.03454999899986433, 0.03531463800027268, 0.037422141002025455, 0.03790182000011555, 0.03617901100005838, 0.03618732199902297, 0.03487651400064351, 0.036152890002995264, 0.03387797300092643, 0.03531463900071685, 0.034897886998805916, 0.0366040749977401, 0.03727253900069627, 0.03284500099834986, 0.03511991800041869, 0.032935237999481615, 0.03481952400034061, 0.03531463900071685, 0.03671330900033354, 0.03501543299717014, 0.03671924599984777, 0.03426266799942823, 0.03196281700002146, 0.03557703899787157, 0.034056073000101605, 0.032747640001616674, 0.03402995199940051, 0.03709087800234556 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-1]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-1]", "params": { "zoom": 8, "item_width": 1 }, "param": "8-1", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.0323878830022295, "max": 0.03815472799760755, "mean": 0.03524561979987387, "stddev": 0.0011380670339047108, "rounds": 30, "median": 0.035330077498656465, "iqr": 0.0012538160008261912, "q1": 0.03457849999904283, "q3": 0.035832315999869024, "iqr_outliers": 2, "stddev_outliers": 8, "outliers": "8;2", "ld15iqr": 0.03317626900025061, "hd15iqr": 0.03815472799760755, "ops": 28.3723198989844, "total": 1.057368593996216, "data": [ 0.0357028979997267, 0.035832315999869024, 0.03457849999904283, 0.03444195699921693, 0.03695196500120801, 0.0339563410016126, 0.03526714899999206, 0.0323878830022295, 0.0353063309994468, 0.03535382399786613, 0.03384117100358708, 0.03504155900009209, 0.03548086899900227, 0.03815472799760755, 0.03578601199842524, 0.035990231997857336, 0.03428760700262501, 0.03587743599928217, 0.03542506499798037, 0.03317626900025061, 0.0349418229998264, 0.03529920800065156, 0.03653284000029089, 0.03491807700265781, 0.035834692000207724, 0.036848668998572975, 0.035555670998292044, 0.03547374499976286, 0.03494300999955158, 0.034180746999481926 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-1.5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-1.5]", "params": { "zoom": 8, "item_width": 1.5 }, "param": "8-1.5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.030248326002038084, "max": 0.03439328300009947, "mean": 0.0319867942187102, "stddev": 0.0009411728636156087, "rounds": 32, "median": 0.031872588999249274, "iqr": 0.0011837645033665467, "q1": 0.03134007299922814, "q3": 0.03252383750259469, "iqr_outliers": 1, "stddev_outliers": 8, "outliers": "8;1", "ld15iqr": 0.030248326002038084, "hd15iqr": 0.03439328300009947, "ops": 31.26290159502964, "total": 1.0235774149987265, "data": [ 0.03385304800031008, 0.03153063799982192, 0.03127417500218144, 0.032710840001527686, 0.033669012002064846, 0.031953325000358745, 0.03182746899983613, 0.03175504200044088, 0.032221661997027695, 0.031555571000353666, 0.03299698599948897, 0.032906748998357216, 0.03243419400314451, 0.031753855997521896, 0.03130860899909749, 0.03439328300009947, 0.032091056997160194, 0.032338019998860545, 0.032613481002044864, 0.03200913099863101, 0.03187615099886898, 0.030248326002038084, 0.030944098998588743, 0.032190791996981716, 0.031371536999358796, 0.031182752001768677, 0.03144158999930369, 0.0310485840018373, 0.03284263400200871, 0.030977344998973422, 0.03186902699962957, 0.030388430001039524 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-2]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-2]", "params": { "zoom": 8, "item_width": 2 }, "param": "8-2", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.028950582000106806, "max": 0.03725355500137084, "mean": 0.031806940118151356, "stddev": 0.0014946978394731498, "rounds": 34, "median": 0.03164403200025845, "iqr": 0.002005395999731263, "q1": 0.030737507000594633, "q3": 0.032742903000325896, "iqr_outliers": 1, "stddev_outliers": 9, "outliers": "9;1", "ld15iqr": 0.028950582000106806, "hd15iqr": 0.03725355500137084, "ops": 31.43967939969577, "total": 1.081435964017146, "data": [ 0.032860447998245945, 0.032930499997746665, 0.030135533001157455, 0.03163987600055407, 0.0333591240014357, 0.03725355500137084, 0.030737507000594633, 0.03192839600160369, 0.03176691900080186, 0.03258973600168247, 0.033474293999461224, 0.0323878910021449, 0.032742903000325896, 0.029632108002260793, 0.03152945599867962, 0.03024951699990197, 0.03317627700016601, 0.030543974000465823, 0.030230520002078265, 0.03336268700149958, 0.03216823700131499, 0.030519040999934077, 0.03144515500025591, 0.03164818799996283, 0.0310153430000355, 0.031651749999582535, 0.03140834799705772, 0.03270609600076568, 0.03134542000043439, 0.030598592002206715, 0.028950582000106806, 0.031205315000988776, 0.03287232200091239, 0.031370354001410306 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-3]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-3]", "params": { "zoom": 8, "item_width": 3 }, "param": "8-3", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.029660606996912975, "max": 0.033832871999038616, "mean": 0.03176537699993898, "stddev": 0.0010999915228235328, "rounds": 33, "median": 0.03196758299964131, "iqr": 0.001908034250845958, "q1": 0.03072385799987387, "q3": 0.03263189225071983, "iqr_outliers": 0, "stddev_outliers": 11, "outliers": "11;0", "ld15iqr": 0.029660606996912975, "hd15iqr": 0.033832871999038616, "ops": 31.480816361849605, "total": 1.0482574409979861, "data": [ 0.033832871999038616, 0.030280389997642487, 0.031687371996667935, 0.032386707000114257, 0.032153991000086535, 0.03178948200002196, 0.03289132200006861, 0.030627090000052704, 0.03196995599864749, 0.0311685109991231, 0.030679332001454895, 0.03199370299989823, 0.03062234100070782, 0.031891594000626355, 0.030738699999346863, 0.03196758299964131, 0.03349804699973902, 0.030088045001321007, 0.0327535930009617, 0.032258478000585455, 0.031011785002192482, 0.03239620800013654, 0.029660606996912975, 0.030378939998627175, 0.031088960997294635, 0.030111790998489596, 0.032640797002386535, 0.03272509700036608, 0.03262892400016426, 0.032580244002019754, 0.03271559900167631, 0.033615592001297045, 0.031423787000676384 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-4]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-4]", "params": { "zoom": 8, "item_width": 4 }, "param": "8-4", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.027282394999929238, "max": 0.03426506200048607, "mean": 0.030401816441072323, "stddev": 0.001716866245843405, "rounds": 34, "median": 0.030336792500747833, "iqr": 0.0023212239975691773, "q1": 0.028907845000503585, "q3": 0.031229068998072762, "iqr_outliers": 0, "stddev_outliers": 12, "outliers": "12;0", "ld15iqr": 0.027282394999929238, "hd15iqr": 0.03426506200048607, "ops": 32.892771454570635, "total": 1.033661758996459, "data": [ 0.030497674997604918, 0.029383961998973973, 0.029731849001109367, 0.03278327899897704, 0.03426506200048607, 0.030776695999520598, 0.03066746300100931, 0.028907845000503585, 0.028470907996961614, 0.030291079998278292, 0.031039095996675314, 0.03215993199773948, 0.029139373000361957, 0.03053567000097246, 0.03009160999863525, 0.031229068998072762, 0.0341736399968795, 0.02858489299978828, 0.030273272001068108, 0.027876059000846, 0.03098210600001039, 0.028721434999170015, 0.031764554001711076, 0.028535025001474423, 0.028791486998670734, 0.03335200999936205, 0.03073989100084873, 0.030266147001384525, 0.03132168100273702, 0.03256718700140482, 0.027282394999929238, 0.028671567000856157, 0.02940533600121853, 0.030382505003217375 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-5]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-5]", "params": { "zoom": 8, "item_width": 5 }, "param": "8-5", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.027861814000061713, "max": 0.03240927499791724, "mean": 0.030260070090937945, "stddev": 0.0009621413396331114, "rounds": 33, "median": 0.030255463996581966, "iqr": 0.001138348749918805, "q1": 0.02963182200164738, "q3": 0.030770170751566184, "iqr_outliers": 1, "stddev_outliers": 8, "outliers": "8;1", "ld15iqr": 0.02877605499816127, "hd15iqr": 0.03240927499791724, "ops": 33.0468500897317, "total": 0.9985823130009521, "data": [ 0.030935801998566603, 0.03057129299850203, 0.029934886002592975, 0.030205596001906088, 0.029239111998322187, 0.02877605499816127, 0.03240927499791724, 0.03159833000245271, 0.029677236001589336, 0.030676965001475764, 0.030527361999702407, 0.03076601500288234, 0.02989332899960573, 0.030255463996581966, 0.031093718000192894, 0.029296104999957606, 0.031021291000797646, 0.03225848599686287, 0.03181442499771947, 0.027861814000061713, 0.02947420399868861, 0.030294647000118857, 0.029697422000026563, 0.02994201100227656, 0.029326976000447758, 0.030294647000118857, 0.030476308002107544, 0.029499137999664526, 0.030782637997617712, 0.030115361001662677, 0.029486077000910882, 0.030704274999152403, 0.02967605000230833 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-6]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-6]", "params": { "zoom": 8, "item_width": 6 }, "param": "8-6", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.028949408999324078, "max": 0.03259212699776981, "mean": 0.030504839750089257, "stddev": 0.0009478542205254766, "rounds": 36, "median": 0.03037479449994862, "iqr": 0.001157642998805386, "q1": 0.029859494999982417, "q3": 0.031017137998787803, "iqr_outliers": 0, "stddev_outliers": 12, "outliers": "12;0", "ld15iqr": 0.028949408999324078, "hd15iqr": 0.03259212699776981, "ops": 32.78168343752974, "total": 1.0981742310032132, "data": [ 0.030885938002029434, 0.029247426999063464, 0.03185717199812643, 0.030361139000888215, 0.029714046002482064, 0.030447814999206457, 0.028966030000447063, 0.03104622699902393, 0.0324805180025578, 0.02979597200101125, 0.03259212699776981, 0.029552569998486433, 0.031183956998575013, 0.02986721200068132, 0.030602167000324698, 0.030076180999458302, 0.02997763299936196, 0.030777891999605345, 0.029904020000685705, 0.032488831002410734, 0.03183461400112719, 0.028949408999324078, 0.029995444001542637, 0.029851777999283513, 0.030988048998551676, 0.030612854003265966, 0.031489102002524305, 0.029530011997849215, 0.03022103599869297, 0.030209163000108674, 0.03128962999835494, 0.029380408999713836, 0.030675781999889296, 0.03028277700286708, 0.030388449999009026, 0.030650847998913378 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-8]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-8]", "params": { "zoom": 8, "item_width": 8 }, "param": "8-8", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.027448631000879686, "max": 0.03276548099893262, "mean": 0.03027509761747805, "stddev": 0.001078552563011638, "rounds": 34, "median": 0.030215102500733337, "iqr": 0.0010638439998729154, "q1": 0.029684368000744144, "q3": 0.03074821200061706, "iqr_outliers": 3, "stddev_outliers": 9, "outliers": "9;3", "ld15iqr": 0.028456670999730704, "hd15iqr": 0.0326989909990516, "ops": 33.03044675973868, "total": 1.0293533189942536, "data": [ 0.03276548099893262, 0.03144517299733707, 0.030481063997285673, 0.031204144997900585, 0.030190167999535333, 0.029652309000084642, 0.030148612000630237, 0.030088057999819284, 0.0326989909990516, 0.03090375200190465, 0.03074821200061706, 0.031194647002848797, 0.03150453899797867, 0.03056773799835355, 0.030211540000891546, 0.02932579400294344, 0.03017354699841235, 0.03028159399764263, 0.030238848998124013, 0.03188804700039327, 0.029565634999016766, 0.03021866500057513, 0.028456670999730704, 0.03047037899887073, 0.02943621700251242, 0.0299527040006069, 0.029684368000744144, 0.030378953997569624, 0.02917262999835657, 0.03024241200182587, 0.02867039000193472, 0.027448631000879686, 0.029856531000405084, 0.03008687200053828 ], "iterations": 1 } }, { "group": "xyzsearch", "name": "test1[8-10]", "fullname": "src/pypgstac/tests/test_benchmark.py::test1[8-10]", "params": { "zoom": 8, "item_width": 10 }, "param": "8-10", "extra_info": {}, "options": { "disable_gc": false, "timer": "perf_counter", "min_rounds": 3, "max_time": 1.0, "min_time": 5e-06, "warmup": 2 }, "stats": { "min": 0.027221853997616563, "max": 0.032586199002253124, "mean": 0.029672734942973227, "stddev": 0.0012590932030563811, "rounds": 35, "median": 0.02938160299891024, "iqr": 0.0017355754989694105, "q1": 0.028647240749705816, "q3": 0.030382816248675226, "iqr_outliers": 0, "stddev_outliers": 10, "outliers": "10;0", "ld15iqr": 0.027221853997616563, "hd15iqr": 0.032586199002253124, "ops": 33.700971680630644, "total": 1.038545723004063, "data": [ 0.027221853997616563, 0.030416951998631703, 0.031562721997033805, 0.030825392001133878, 0.02917500699913944, 0.029343607002374483, 0.028673954999248963, 0.0297544229979394, 0.027784646998043172, 0.028380685002048267, 0.03194741600236739, 0.03218013200239511, 0.02952764300061972, 0.031147157002124004, 0.030203231999621494, 0.029350731001613894, 0.02928305300156353, 0.028575405998708447, 0.028595591000339482, 0.029159572997741634, 0.02995151999857626, 0.0286383359998581, 0.029634503000124823, 0.030280408998805797, 0.02938160299891024, 0.03072565800175653, 0.032586199002253124, 0.0293721039997763, 0.028415119002602296, 0.03014505399914924, 0.02973305200066534, 0.028434116000426002, 0.03099636800106964, 0.028245330999197904, 0.028897173000586918 ], "iterations": 1 } } ], "datetime": "2025-01-07T03:09:36.802381+00:00", "version": "5.1.0" } ================================================ FILE: docs/src/item_size_analysis.ipynb ================================================ { "cells": [ { "cell_type": "markdown", "id": "f7b04830-7b6a-4e37-a2a3-9961970e06df", "metadata": {}, "source": [ "# impacts of STAC item footprint size on dynamic tiling query performance\n", "\n", "**TL;DR:** If you have any control over the geographic footprint of the assets that you are cataloging with `pgstac` and you want to serve visualizations with a dynamic tiling application, try to maximize the size of the assets!\n", "\n", "Dynamic tiling applications like [`titiler-pgstac`](https://github.com/stac-utils/titiler-pgstac) send many queries to a `pgstac` database and clients are very sensitive to performance so it is worth considering a few basic ideas when building collections and items that may be used in this way.\n", "\n", "`pgstac`'s query functions perform relatively expensive spatial intersection operations so the fewer items there are in a collection x datetime partition, the faster the query will be. This is not a `pgstac`-specific problem (any application that needs to perform spatial intersections will take longer as the number of calculations increases), but it is worth demonstrating the influence of these factors in the dynamic tiling context." ] }, { "cell_type": "code", "execution_count": null, "id": "d34feaea-5288-4124-bca1-6bd4090fd27d", "metadata": {}, "outputs": [], "source": [ "import json\n", "import math\n", "import uuid\n", "from datetime import datetime, timezone\n", "from typing import Any, Dict, Generator, Tuple\n", "\n", "import numpy as np\n", "import pandas as pd\n", "import seaborn as sns\n", "from folium import Map, GeoJson, LayerControl\n", "from matplotlib.colors import LogNorm\n", "\n", "\n", "XMIN, YMIN = 0, 0\n", "AOI_WIDTH = 50\n", "AOI_HEIGHT = 50\n", "ITEM_WIDTHS = [0.5, 1, 2, 4, 6, 8, 10]\n", "\n", "def generate_items(\n", " item_size: Tuple[float, float],\n", " collection_id: str,\n", ") -> Generator[Dict[str, Any], None, None]:\n", " item_width, item_height = item_size\n", "\n", " cols = math.ceil(AOI_WIDTH / item_width)\n", " rows = math.ceil(AOI_HEIGHT / item_height)\n", "\n", " # Generate items for each grid cell\n", " for row in range(rows):\n", " for col in range(cols):\n", " left = XMIN + (col * item_width)\n", " bottom = YMIN + (row * item_height)\n", " right = left + item_width\n", " top = bottom + item_height\n", "\n", " yield {\n", " \"type\": \"Feature\",\n", " \"stac_version\": \"1.0.0\",\n", " \"id\": str(uuid.uuid4()),\n", " \"collection\": collection_id,\n", " \"geometry\": {\n", " \"type\": \"Polygon\",\n", " \"coordinates\": [\n", " [\n", " [left, bottom],\n", " [right, bottom],\n", " [right, top],\n", " [left, top],\n", " [left, bottom],\n", " ],\n", " ],\n", " },\n", " \"bbox\": [left, bottom, right, top],\n", " \"properties\": {\n", " \"datetime\": datetime.now(timezone.utc).isoformat(),\n", " },\n", " }\n", "\n", "\n", "\n", "def load_benchmark_results() -> pd.DataFrame:\n", " \"\"\"Load benchmark results from JSON file into a pandas DataFrame.\"\"\"\n", " with open(\"./benchmark.json\") as f:\n", " data = json.load(f)\n", "\n", " # Extract the benchmarks into a list of records\n", " records = []\n", " for benchmark in data[\"benchmarks\"]:\n", " record = {\n", " \"item_width\": benchmark[\"params\"][\"item_width\"],\n", " \"zoom\": benchmark[\"params\"][\"zoom\"],\n", " \"mean\": benchmark[\"stats\"][\"mean\"],\n", " \"stddev\": benchmark[\"stats\"][\"stddev\"],\n", " \"median\": benchmark[\"stats\"][\"median\"],\n", " }\n", "\n", " records.append(record)\n", "\n", " return pd.DataFrame(records).sort_values(by=[\"item_width\", \"zoom\"])\n", "\n", "\n", "stac_items = {\n", " item_width: list(\n", " generate_items(\n", " (item_width, item_width),\n", " f\"{item_width} degrees\"\n", " )\n", " )\n", " for item_width in ITEM_WIDTHS\n", "}\n", "\n", "df = load_benchmark_results()" ] }, { "cell_type": "markdown", "id": "ceb365dc-67c1-4cbd-8b5e-7022a5773140", "metadata": {}, "source": [ "## Scenario\n", "Imagine you have a continental-scale dataset of gridded data that will be stored as cloud-optimized geotiffs (COGs) and you get to decide how the individual files will be spatially arranged and cataloged in a `pgstac` database. You could make items as small as 0.5 degree squares or as large as 10 degree squares. In this case the assets will be non-overlapping rectangular grids.\n", "\n", "The assets will be publicly accessible, so smaller file sizes might be useful for some applications/users, but since the data will be stored as COGs and we also want to be able to serve raster tile visualizations in a web map with `titiler-pgstac`, smaller file sizes are not very important. However, the processing pipleline that generates the assets might have some resource constraints that push you to choose a smaller tile size.\n", "\n", "Consider the following options for tile sizes:" ] }, { "cell_type": "code", "execution_count": null, "id": "7ae1604f-df80-485b-a02a-4237f9ab0081", "metadata": {}, "outputs": [], "source": [ "pd.DataFrame(\n", " {\"tile width (degrees)\": item_width, \"# items\": len(items)}\n", " for item_width, items in stac_items.items()\n", ")" ] }, { "cell_type": "markdown", "id": "fce001cb-ceb8-458e-9bda-e207d20362e7", "metadata": {}, "source": [ "The number of items is inversely proportional to the square of the tile width which means that small changes in tile size can have a large impact on the eventual number of items in your catalog!\n", "\n", "This map shows the spatial arrangement of the items for a range of tile sizes:" ] }, { "cell_type": "code", "execution_count": null, "id": "4c19eafb-125e-46ad-8b6f-e0053084287f", "metadata": {}, "outputs": [], "source": [ "m = Map([25, 25], zoom_start=3)\n", "for item_width in ITEM_WIDTHS:\n", " layer_name = f\"{item_width} degrees\"\n", " geojson = GeoJson(\n", " {\n", " \"type\": \"FeatureCollection\",\n", " \"features\": stac_items[item_width],\n", " },\n", " name=layer_name,\n", " overlay=True,\n", " show=False,\n", " )\n", " geojson.add_to(m)\n", " \n", "LayerControl(collapsed=False, position=\"topright\").add_to(m)\n", "\n", "m" ] }, { "cell_type": "markdown", "id": "1c9714e7-c865-4b73-8305-83851864e486", "metadata": {}, "source": [ "## Performance comparison\n", "To simulate the performance of queries made by a dynamic tiling application we have prepared a benchmarking procedure that uses the `pgstac` function `xyzsearch` to run an item query for an XYZ tile. By iterating over many combinations of tile sizes and zoom levels we can examine the response time with respect to item footprint size and tile zoom level. \n", "\n", "This figure shows average response time for `xyzsearch` to return a complete set of results for each zoom level for the range of item tile widths:" ] }, { "cell_type": "code", "execution_count": null, "id": "c61e8c44-df19-404d-8878-1efb5fddeb36", "metadata": {}, "outputs": [], "source": [ "ax = sns.heatmap(\n", " df.pivot(index=\"item_width\", columns=\"zoom\", values=\"median\"),\n", " norm=LogNorm(vmin=1e-2, vmax=1e1),\n", " cbar_kws={\n", " \"ticks\": np.logspace(-2, 0, num=3),\n", " \"format\": \"%.1e\",\n", " }\n", ")\n", "ax.set(xlabel=\"zoom level\", ylabel=\"item tile width\")\n", "ax.xaxis.tick_top()\n", "ax.xaxis.set_label_position(\"top\")\n", "display(ax)" ] }, { "cell_type": "markdown", "id": "cdb98e2e-e9da-4516-85f7-5576035b5915", "metadata": {}, "source": [ "Without details about the resource configuration for a specific `pgstac` deployment it is hard to say which zoom level becomes inoperable for a given tile size, but queries that take >0.5 seconds in this test would probably yield poor results in a deployed context." ] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.3" } }, "nbformat": 4, "nbformat_minor": 5 } ================================================ FILE: docs/src/pgstac.md ================================================ PGDatabase Schema and Functions for Storing and Accessing STAC collections and items in PostgreSQL STAC Client that uses PgSTAC available in [STAC-FastAPI](https://github.com/stac-utils/stac-fastapi) PgSTAC requires **Postgresql>=13**, **PostGIS>=3** and **btree_gist**. Best performance will be had using PostGIS>=3.1. ### PgSTAC Settings PgSTAC installs everything into the pgstac schema in the database. This schema must be in the search_path in the postgresql session while using pgstac. #### PgSTAC Users The `pgstac_admin` role is the owner of all the objects within pgstac and should be used when running things such as migrations. The `pgstac_ingest` role has read/write privileges on all tables and should be used for data ingest or if using the transactions extension with stac-fastapi-pgstac. The `pgstac_read` role has read only access to the items and collections, but will still be able to write to the logging tables. You can use the roles either directly and adding a password to them or by granting them to a role you are already using. To use directly: ```sql ALTER ROLE pgstac_read LOGIN PASSWORD ''; ``` To grant pgstac permissions to a current postgresql user: ```sql GRANT pgstac_read TO ; ``` #### PgSTAC Search Path The search_path can be set at the database level or role level or by setting within the current session. The search_path is already set if you are directly using one of the pgstac users. If you are not logging in directly as one of the pgstac users, you will need to set the search_path by adding it to the search_path of the user you are using: ```sql ALTER ROLE SET SEARCH_PATH TO pgstac, public; ``` setting the search_path on the database: ```sql ALTER DATABASE set search_path to pgstac, public; ``` In psycopg the search_path can be set by passing it as a configuration when creating your connection: ```python kwargs={ "options": "-c search_path=pgstac,public" } ``` #### PgSTAC Settings Variables There are additional variables that control the settings used for calculating and displaying context (total row count) for a search, as well as a variable to set the filter language (cql-json or cql2-json). The context is "off" by default, and the default filter language is set to "cql2-json". Variables can be set either by passing them in via the connection options using your connection library, setting them in the pgstac_settings table or by setting them on the Role that is used to log in to the database. Turning "context" on can be **very** expensive on larger databases. Much of what PgSTAC does is to optimize the search of items sorted by time where only fewer than 10,000 records are returned at a time. It does this by searching for the data in chunks and is able to "short circuit" and return as soon as it has the number of records requested. Calculating the context (the total count for a query) requires a scan of all records that match the query parameters and can take a very long time. Setting "context" to auto will use database statistics to estimate the number of rows much more quickly, but for some queries, the estimate may be quite a bit off. Example for updating the pgstac_settings table with a new value: ```sql INSERT INTO pgstac_settings (name, value) VALUES ('default-filter-lang', 'cql-json'), ('context', 'on') ON CONFLICT ON CONSTRAINT pgstac_settings_pkey DO UPDATE SET value = excluded.value; ``` Alternatively, update the role: ```sql ALTER ROLE SET SEARCH_PATH to pgstac, public; ALTER ROLE SET pgstac.context TO <'on','off','auto'>; ALTER ROLE SET pgstac.context_estimated_count TO ''; ALTER ROLE SET pgstac.context_estimated_cost TO ''; ALTER ROLE SET pgstac.context_stats_ttl TO '>'; ``` The check_pgstac_settings function can be used to check what pgstac settings are being used and to check recommendations for system settings. It takes a single parameter which should be the amount of memory available on the database system. ```sql SELECT check_pgstac_settings('16GB'); ``` ##### Read Only Mode The pgstac.readonly setting can be used when using pgstac with a read replica. Note that when pgstac.readonly is set to TRUE that pgstac is unable to use a cache for calculating the total count for context which can make use of the context extension very expensive (see notes above). In readonly mode, pgstac is also unable to register the hash that is used to store queries that can be used with geometry_search (used by titiler-pgstac). A registered hash will still be readable, but new hashes cannot be created on the read only replica, they must be registered on the main database. #### Runtime Configurations Runtime configuration of variables can be made with search by passing in configuration in the search json "conf" item. Runtime configuration is available for **context**, **context_estimated_count**, **context_estimated_cost**, **context_stats_ttl**, and **nohydrate**. The nohydrate conf item returns an unhydrated item bypassing the CPU intensive step of rehydrating data with data from the collection metadata. When using the nohydrate conf, the only fields that are respected in the fields extension are geometry and bbox. ```sql SELECT search('{"conf":{"nohydrate"=true}}'); ``` #### PgSTAC Partitioning By default PgSTAC partitions data by collection (note: this is a change starting with version 0.5.0). Each collection can further be partitioned by either year or month. **Partitioning must be set up prior to loading any data!** Partitioning can be configured by setting the partition_trunc flag on a collection in the database. ```sql UPDATE collections set partition_trunc='month' WHERE id=''; ``` In general, you should aim to keep each partition less than a few hundred thousand rows. Further partitioning (ie setting everything to 'month' when not needed to keep the partitions below a few hundred thousand rows) can be detrimental. #### PgSTAC Indexes / Queryables By default, PgSTAC includes indexes on the id, datetime, collection, and geometry. Further indexing can be added for additional properties globally or only on particular collections by modifications to the queryables table. The `queryables` table controls the indexes that PgSTAC will build as well as the metadata that is returned from a [STAC Queryables endpoint](https://github.com/stac-api-extensions/filter#queryables). | Column | Description | Type | Example | |-----------------------|--------------------------------------------------------------------------|------------|--------------------------------------------------------------------------------------------------------------------| | `id` | The id of the queryable | bigint[pk] | - | | `name` | The name of the property | text | `eo:cloud_cover` | | `collection_ids` | The collection ids that this queryable applies to | text[] | `{sentinel-2-l2a,landsat-c2-l2,aster-l1t}` or `NULL` | | `definition` | The queryable definition of the property | jsonb | `{"title": "Cloud Cover", "type": "number", "minimum": 0, "maximum": 100}` | | `property_wrapper` | The wrapper function to use to convert the property to a searchable type | text | One of `to_int`, `to_float`, `to_tstz`, `to_text` or `NULL` | | `property_index_type` | The index type to use for the property | text | `BTREE`, `NULL` or other valid [PostgreSQL index type](https://www.postgresql.org/docs/current/indexes-types.html) | Each record in the queryables table references a single property but can apply to any number of collections. If the `collection_ids` field is left as NULL, then that queryable will apply to all collections. There are constraints that allow only a single queryable record to be active per collection. If there is a queryable already set for a property field with collection_ids set to NULL, you will not be able to create a separate queryable entry that applies to that property with a specific collection as pgstac would not then be able to determine which queryable entry to use. By default, any property may be used in filter expressions. If you wish to restrict it and only allow the queryables, you should either set the additional_properties setting variable to False or make the corresponding adjustment in the pgstac_settings table. ##### Queryable Metadata When used with [stac-fastapi](https://stac-utils.github.io/stac-fastapi/), the metadata returned in the queryables endpoint is determined using the definition field on the `queryables` table. This is a jsonb field that will be returned as-is in the queryables response. The full queryable response for a collection will be determined by all the `queryables` records that have a match in `collection_ids` or have a NULL `collection_ids`. If two or more collections in your catalog share a property name, but have different definitions (e.g., `platform` with different enum values), be sure to repeat the property for each collection id, each with a unique `definition`. There is a utility SQL function that can be used to help populate the `queryables` table by looking at a sample of data for each collection. This utility can also look to the json schema for STAC extensions defined in the `stac_extensions` table. The `stac_extensions` table contains a `url` field and a `content` field for each extension that should be introspected to compare for fields. This can either be filled in manually or by using the `pypgstac loadextensions` command included with pypgstac. This command will look at the `stac_extensions` attribute in all collections to populate the `stac_extensions` table, fetching the json content of each extension. If any urls were added manually to the stac_extensions table, it will also populate any records where the content is NULL. Once the `stac_extensions` table has been filled in, you can run the `missing_queryables` function either for a single collection: ```sql SELECT * FROM missing_queryables('mycollection', 5); ``` or for all collections: ```sql SELECT * FROM missing_queryables(5); ``` The numeric argument is the approximate percent of items that should be sampled to look for fields to include. This function will look for fields in the properties of items that do not already exist in the queryables table for each collection. It will then look to see if there is a field in any definition in the stac_extensions table to populate the definition for the queryable. If no definition was found, it will use the data type of the values for that field in the sample of items to fill in a generic definition with just the field type. In order to populate the queryables table, you can then run the following query. Note we're casting the collection id to a text array: ```sql INSERT INTO queryables (collection_ids, name, definition, property_wrapper) SELECT array[collection]::text[] as collection_ids, name, definition, property_wrapper FROM missing_queryables('mycollection', 5) ``` If you run into conflicts due to the unique constraints on collection/name, you may need to create a temp table, make any changes to remove the conflicts, and then INSERT. ```sql CREATE TEMP TABLE draft_queryables AS SELECT * FROM missing_queryables(5); ``` Make any edits to that table or the existing queryables, then: ```sql INSERT INTO queryables (collection_ids, name, definition, property_wrapper) SELECT * FROM draft_queryables; ``` ##### Indexing The `queryables` table is also used to specify which item `properties` attributes to add indexes on. To add a new global index across all collection partitions: ```sql INSERT INTO pgstac.queryables (name, property_wrapper, property_index_type) VALUES (, , ); ``` Property wrapper should be one of `to_int`, `to_float`, `to_tstz`, or `to_text`. The index type should almost always be `BTREE`, but can be any PostgreSQL index type valid for the data type. **More indexes is not necessarily better.** You should only index the primary fields that are actively being used to search. Adding too many indexes can be very detrimental to performance and ingest speed. If your primary use case is delivering items sorted by datetime and you do not use the context extension, you likely will not need any further indexes. Leave `property_index_type` set to NULL if you do not want an index set for a property. ### Maintenance Procedures These are procedures that should be run periodically to make sure that statistics and constraints are kept up-to-date and validated. These can be made to run regularly using the pg_cron extension if available. ```sql SELECT cron.schedule('0 * * * *', 'CALL validate_constraints();'); SELECT cron.schedule('10, * * * *', 'CALL analyze_items();'); ``` ### System Checks #### System and pgSTAC Settings PgSTAC includes a function for checking common Postgres settings. You must pass in the amount of total memory available to get appropriate memory settings. Do note that these suggestions are merely a good first pass and figuring out the ideal parameters requires further analysis of a system and logs. ```sql SELECT * FROM check_pgstac_settings('32GB'); ``` #### Unused Indexes Having too many indexes that do not get used can drastically hurt the performance of the database. The following query can be used to show indexes that are not getting used and should be considered for removal. ```sql SELECT relname, indexrelname, idx_scan, last_idx_scan,idx_tup_read, idx_tup_fetch, pg_relation_size(relid) as index_bytes, pg_size_pretty(pg_relation_size(relid)) as index_size FROM pg_stat_user_indexes WHERE schemaname = 'pgstac' ORDER BY idx_scan ASC ; ``` ### Backup and Restore (pg_dump / pg_restore) PgSTAC databases can be backed up and restored using standard PostgreSQL `pg_dump` and `pg_restore` tools. The custom format (`-Fc`) is recommended for flexibility and compression. #### Dumping Since PgSTAC installs everything into the `pgstac` schema, dump only that schema: ```bash pg_dump -Fc --schema=pgstac -f pgstac_backup.dump mydatabase ``` This captures all tables, functions, views, indexes, and data within the `pgstac` schema. #### Restoring with `pgstac_restore` **Important:** `pg_restore` cannot be used directly with PgSTAC dumps. `pg_dump` sets `search_path` to empty during restore for safety, but PgSTAC SQL functions reference PostGIS functions (e.g. `st_makeenvelope`, `st_geomfromgeojson`) without schema qualification. Since PostGIS may be installed in either the `public` or `postgis` schema depending on the deployment, PgSTAC does not schema-qualify these calls. This means a raw `pg_restore` will fail when creating functions and tables that depend on PostGIS. PgSTAC provides the `pgstac_restore` script to handle this. It installs a temporary event trigger on the target database that sets the correct `search_path` (including the detected PostGIS schema) before each DDL command, then runs `pg_restore` directly: ```bash pgstac_restore -d target_database pgstac_backup.dump ``` The script accepts several options: - `--create-extensions` — Create PostGIS, btree_gist, and unaccent on the target if they don't exist - `--no-roles` — Skip ownership and privilege restoration (use when the target doesn't have `pgstac_admin`/`pgstac_read`/`pgstac_ingest` roles) - `-d, --dbname` — Target database name (also reads `PGDATABASE`) Common restore patterns: ```bash # Restore into a fresh database, creating extensions, without requiring pgstac roles pgstac_restore -d target_db --create-extensions --no-roles pgstac_backup.dump # Restore into a database that already has extensions and pgstac roles pgstac_restore -d target_db pgstac_backup.dump ``` After restoring, ensure the `search_path` is set on the target database or role: ```sql ALTER DATABASE target_database SET search_path TO pgstac, public; ``` #### Notes - Always use `--schema=pgstac` on `pg_dump` to capture only the pgstac schema. - Do **not** use `pg_restore` directly — use `pgstac_restore` instead to handle the search_path issue. - After restoring, you may want to run `ANALYZE` on the restored database to update planner statistics. - Materialized views (`partitions`, `partition_steps`) are included in the dump. If you need to refresh them after restore, run `REFRESH MATERIALIZED VIEW pgstac.partitions; REFRESH MATERIALIZED VIEW pgstac.partition_steps;`. ### Notification Triggers You can add notification triggers alongside pgstac to get notified when items are inserted, updated, or deleted. **Important:** Do NOT create these in the `pgstac` schema as they could be removed in future migrations. **Note on pgSTAC's UPDATE behavior:** For performance and consistency, `update_item()` in pgSTAC uses a **DELETE+INSERT** pattern instead of a direct SQL `UPDATE`. As a result, standard `UPDATE` triggers will not fire during these operations. Applications that rely on change notifications should implement logic that accounts for this behavior. #### Example Notification Setup Here's an example of how to set up notification triggers for item changes: ```sql -- Create the notification function CREATE OR REPLACE FUNCTION notify_items_change_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM pg_notify('pgstac_items_change'::text, json_build_object( 'operation', TG_OP, 'items', jsonb_agg( jsonb_build_object( 'collection', data.collection, 'id', data.id ) ) )::text ) FROM data ; RETURN NULL; END; $$ LANGUAGE plpgsql; -- Create triggers for INSERT operations CREATE OR REPLACE TRIGGER notify_items_change_insert AFTER INSERT ON pgstac.items REFERENCING NEW TABLE AS data FOR EACH STATEMENT EXECUTE FUNCTION notify_items_change_func() ; -- Create triggers for UPDATE operations CREATE OR REPLACE TRIGGER notify_items_change_update AFTER UPDATE ON pgstac.items REFERENCING NEW TABLE AS data FOR EACH STATEMENT EXECUTE FUNCTION notify_items_change_func() ; -- Create triggers for DELETE operations CREATE OR REPLACE TRIGGER notify_items_change_delete AFTER DELETE ON pgstac.items REFERENCING OLD TABLE AS data FOR EACH STATEMENT EXECUTE FUNCTION notify_items_change_func() ; ``` #### Usage Listen for notifications: ```sql LISTEN pgstac_items_change; ``` Payload structure: ```json { "operation": "INSERT", "items": [ { "collection": "sentinel-2-l2a", "id": "item-1" }, { "collection": "sentinel-2-l2a", "id": "item-2" } ] } ``` #### Customization You may modify the function to include additional metadata, filter by collection, or use different channels. Only trigger on the `items` table, not `items_staging`. ================================================ FILE: docs/src/pypgstac.md ================================================ PgSTAC includes a Python utility for bulk data loading and managing migrations. pyPgSTAC is available on PyPI ``` python -m pip install pypgstac ``` By default, pyPgSTAC does not install the `psycopg` dependency. If you want the database driver installed, use: ``` python -m pip install pypgstac[psycopg] ``` Or can be built locally ``` git clone https://github.com/stac-utils/pgstac cd pgstac/pypgstac python -m pip install . ``` ``` pypgstac --help Usage: pypgstac [OPTIONS] COMMAND [ARGS]... Options: --install-completion Install completion for the current shell. --show-completion Show completion for the current shell, to copy it or customize the installation. --help Show this message and exit. Commands: initversion Get initial version. load Load STAC data into a pgstac database. migrate Migrate a pgstac database. pgready Wait for a pgstac database to accept connections. version Get version from a pgstac database. ``` pyPgSTAC will get the database connection settings from the **standard PG environment variables**: - PGHOST=0.0.0.0 - PGPORT=5432 - PGUSER=username - PGDATABASE=postgis - PGPASSWORD=asupersecretpassword It can also take a DSN database url "postgresql://..." via the **--dsn** flag. ### Migrations pyPgSTAC has a utility to help apply migrations to an existing PgSTAC instance to bring it up to date. There are two types of migrations: - **Base migrations** install PgSTAC into a database with no current PgSTAC installation. These migrations follow the file pattern `"pgstac.[version].sql"` - **Incremental migrations** are used to move PgSTAC from one version to the next. These migrations follow the file pattern `"pgstac.[version].[fromversion].sql"` Migrations are stored in ```pypgstac/pypgstac/migrations``` and are distributed with the pyPgSTAC package. ### Running Migrations pyPgSTAC has a utility for checking the version of an existing PgSTAC database and applying the appropriate migrations in the correct order. It can also be used to setup a database from scratch. To create an initial PgSTAC database or bring an existing one up to date, check you have the pypgstac version installed you want to migrate to and run: ``` pypgstac migrate ``` ### Bootstrapping an Empty Database When starting with an empty database, you have two options for initializing PgSTAC: #### Option 1: Execute as Power User This approach uses a database user with administrative privileges (such as 'postgres') to run the migration, which will automatically create all necessary extensions and roles: ```bash # Set environment variables for database connection export PGHOST=localhost export PGPORT=5432 export PGDATABASE=yourdatabase export PGUSER=postgres # A user with admin privileges export PGPASSWORD=yourpassword # Run the migration pypgstac migrate ``` The migration process will automatically: - Create required extensions (postgis, btree_gist, unaccent) - Create necessary roles (pgstac_admin, pgstac_read, pgstac_ingest) - Set up the pgstac schema and tables In production environments, you should assign these roles to your application database user rather than continuing to use the postgres user: ```sql -- Grant appropriate roles to your application user GRANT pgstac_read TO your_app_user; GRANT pgstac_ingest TO your_app_user; GRANT pgstac_admin TO your_app_user; -- Set the search path for your application user ALTER USER your_app_user SET search_path TO pgstac, public; ``` #### Option 2: Create User with Initial Grants If you don't have administrative privileges or prefer more control over the setup process, you can manually prepare the database before running migrations. Connect to your database as an administrator and execute: ```sql \c [database] -- Create required extensions CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; CREATE EXTENSION IF NOT EXISTS unaccent; -- Create required roles CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; -- Grant appropriate permissions ALTER DATABASE [database] OWNER TO [user]; ALTER USER [user] SET search_path TO pgstac, public; ALTER DATABASE [database] set search_path to pgstac, public; GRANT CONNECT ON DATABASE [database] TO [user]; GRANT ALL PRIVILEGES ON TABLES TO [user]; GRANT ALL PRIVILEGES ON SEQUENCES TO [user]; GRANT pgstac_read TO [user] WITH ADMIN OPTION; GRANT pgstac_ingest TO [user] WITH ADMIN OPTION; GRANT pgstac_admin TO [user] WITH ADMIN OPTION; ``` Then run the migration as your non-admin user: ```bash # Set environment variables for database connection export PGHOST=localhost export PGPORT=5432 export PGDATABASE=yourdatabase export PGUSER=[user] # Your non-admin user export PGPASSWORD=yourpassword # Run the migration pypgstac migrate ``` ### Verifying Migration To verify that PgSTAC was installed correctly: ```bash # Check the PgSTAC version pypgstac version ``` ### Bulk Data Loading A python utility is included which allows to load data from any source openable by smart-open using python in a memory efficient streaming manner using PostgreSQL copy. There are options for collections and items and can be used either as a command line or a library. To load an ndjson of items directly using copy (will fail on any duplicate ids but is the fastest option to load new data you know will not conflict) ``` pypgstac load items ``` To load skipping any records that conflict with existing data ``` pypgstac load items --method insert_ignore ``` To upsert any records, adding anything new and replacing anything with the same id ``` pypgstac load items --method upsert ``` ### Loading Queryables Queryables are a mechanism that allows clients to discover what terms are available for use when writing filter expressions in a STAC API. The Filter Extension enables clients to filter collections and items based on their properties using the Common Query Language (CQL2). To load queryables from a JSON file: ``` pypgstac load_queryables queryables.json ``` To load queryables for specific collections: ``` pypgstac load_queryables queryables.json --collection_ids [collection1,collection2] ``` To load queryables and delete properties not present in the file: ``` pypgstac load_queryables queryables.json --delete_missing ``` To load queryables and create indexes only for specific fields: ``` pypgstac load_queryables queryables.json --index_fields [field1,field2] ``` By default, no indexes are created when loading queryables. Using the `--index_fields` parameter allows you to selectively create indexes only for fields that require them. Creating too many indexes can degrade database performance, especially for write operations, so it's recommended to only index fields that are frequently used in queries. When using `--delete_missing` with specific collections, only properties for those collections will be deleted: ``` pypgstac load_queryables queryables.json --collection_ids [collection1,collection2] --delete_missing ``` You can combine all parameters as needed: ``` pypgstac load_queryables queryables.json --collection_ids [collection1,collection2] --delete_missing --index_fields [field1,field2] ``` The JSON file should follow the queryables schema as described in the [STAC API - Filter Extension](https://github.com/stac-api-extensions/filter#queryables). Here's an example: ```json { "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://example.com/stac/queryables", "type": "object", "title": "Queryables for Example STAC API", "description": "Queryable names for the Example STAC API", "properties": { "id": { "description": "Item identifier", "type": "string" }, "datetime": { "description": "Datetime", "type": "string", "format": "date-time" }, "eo:cloud_cover": { "description": "Cloud cover percentage", "type": "number", "minimum": 0, "maximum": 100 } }, "additionalProperties": true } ``` The command will extract the properties from the JSON file and create queryables in the database. It will also determine the appropriate property wrapper based on the type of each property and create the necessary indexes. ### Automated Collection Extent Updates By setting `pgstac.update_collection_extent` to `true`, a trigger is enabled to automatically adjust the spatial and temporal extents in collections when new items are ingested. This feature, while helpful, may increase overhead within data load transactions. To alleviate performance impact, combining this setting with `pgstac.use_queue` is beneficial. This approach necessitates a separate process, such as a scheduled task via the `pg_cron` extension, to periodically invoke `CALL run_queued_queries();`. Such asynchronous processing ensures efficient transactional performance and updated collection extents. *Note: The `pg_cron` extension must be properly installed and configured to manage the scheduling of the `run_queued_queries()` function.* ================================================ FILE: scripts/cibuild ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. set -e if [[ "${CI}" ]]; then set -x fi function usage() { echo -n \ "Usage: $(basename "$0") CI build for this project. " } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then scripts/setup scripts/test fi ================================================ FILE: scripts/cipublish ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. set -e if [[ -n "${CI}" ]]; then set -x fi function usage() { echo -n \ "Usage: $(basename "$0") Publish pypgstac Options: --test Publish to test pypi " } POSITIONAL=() while [[ $# -gt 0 ]] do key="$1" case $key in --help) usage exit 0 shift ;; --test) TEST_PYPI="--repository testpypi" shift ;; *) # unknown option POSITIONAL+=("$1") # save it in an array for later shift # past argument ;; esac done set -- "${POSITIONAL[@]}" # restore positional parameters # Fail if this isn't CI and we aren't publishing to test pypi if [ -z "${TEST_PYPI}" ] && [ -z "${CI}" ]; then echo "Only CI can publish to pypi" exit 1 fi if [ "${BASH_SOURCE[0]}" = "${0}" ]; then pushd pypgstac rm -rf dist pip install build twine python -m build --sdist --wheel twine upload ${TEST_PYPI} dist/* popd fi ================================================ FILE: scripts/console ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. source "$SCRIPT_DIR/pgstacenv" set -e if [[ "${CI}" ]]; then set -x fi function usage() { echo -n \ "Usage: $(basename "$0") [--db] [--help] Start a console in the dev container --db: Instead, start a psql console in the database container. " } while [[ "$#" > 0 ]]; do case $1 in --db) DB_CONSOLE=1 shift ;; -h|--help) usage exit 0 ;; *) echo "Unknown parameter passed: $1" >&2 usage exit 1 ;; esac; done if [ "${BASH_SOURCE[0]}" = "${0}" ]; then ensure_env_file docker compose up -d if [[ "${DB_CONSOLE}" ]]; then docker compose exec pgstac psql exit 0 fi docker compose exec pypgstac /bin/bash fi ================================================ FILE: scripts/container-scripts/format ================================================ #!/bin/bash set -e if [[ "${CI}" ]]; then set -x fi SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd ${PGSTAC_REPO_DIR:-/opt/src} function usage() { echo -n \ "Usage: $(basename "$0") Format code. This script is meant to be run inside the dev container. " } if [ "${BASH_SOURCE[0]}" = "${0}" ]; then echo "Formatting pypgstac..." ruff check --fix pypgstac/src/pypgstac pypgstac/tests ruff format pypgstac/src/pypgstac pypgstac/tests fi ================================================ FILE: scripts/container-scripts/initpgstac ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd ${PGSTAC_PGSTAC_DIR:-/opt/src/pgstac} psql -X -q -v ON_ERROR_STOP=1 < /dev/null && pwd ) cd ${PGSTAC_PGSTAC_DIR:-/opt/src/pgstac} psql -f pgstac.sql psql -v ON_ERROR_STOP=1 <<-EOSQL \copy collections (content) FROM 'tests/testdata/collections.ndjson' \copy items_staging (content) FROM 'tests/testdata/items.ndjson' EOSQL ================================================ FILE: scripts/container-scripts/makemigration ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) SRCDIR=${PGSTAC_REPO_DIR:-/opt/src} cd $SRCDIR SHORT=f:,t:,o,d,h LONG=from:,to:,overwrite,debug,help OPTS=$(getopt --alternative --name $0 --options $SHORT --longoptions $LONG -- "$@") eval set -- "$OPTS" while : do case "$1" in -f | --from ) FROM="$2" shift 2 ;; -t | --to ) TO="$2" shift 2 ;; -o | --overwrite ) OVERWRITE=1 shift 1 ;; -d | --debug ) DEBUG=1 shift 1 ;; -h | --help) cat <&2 exit 1 fi BASEDIR=$SRCDIR PYPGSTACDIR=$BASEDIR/pypgstac MIGRATIONSDIR=$BASEDIR/pgstac/migrations SQLDIR=$BASEDIR/pgstac/sql # Check if from SQL file exists FROMSQL=$MIGRATIONSDIR/pgstac.$FROM.sql if [ -f $FROMSQL ]; then echo "Migrating From: $FROMSQL" else echo "From SQL $FROMSQL does not exist" exit 1 fi # Check if to SQL file exists TOSQL=$MIGRATIONSDIR/pgstac.$TO.sql if [ -f $TOSQL ]; then echo "Migrating To: $TOSQL" else echo "To SQL $TOSQL does not exist" exit 1 fi MIGRATIONSQL=$MIGRATIONSDIR/pgstac.$FROM-$TO.sql if [[ -f "$MIGRATIONSQL" ]]; then if [[ "$OVERWRITE" != 1 ]]; then echo "ERROR: $MIGRATIONSQL already exists. Use --overwrite to replace." >&2 exit 1 else echo "Removing existing $MIGRATIONSQL" rm $MIGRATIONSQL fi else echo "Creating $MIGRATIONSQL" fi pg_isready -t 10 # Create Databases to inspect to create migration psql -q >/dev/null 2>&1 <<-'EOSQL' DROP DATABASE IF EXISTS migra_from; CREATE DATABASE migra_from; DROP DATABASE IF EXISTS migra_to; CREATE DATABASE migra_to; EOSQL TODBURL="postgresql://${PGUSER}:${PGPASSWORD}@${PGHOST:-localhost}:${PGPORT:-5432}/migra_to" FROMDBURL="postgresql://${PGUSER}:${PGPASSWORD}@${PGHOST:-localhost}:${PGPORT:-5432}/migra_from" # Make sure to clean up migra databases function drop_migra_dbs(){ psql -q >/dev/null 2>&1 <<-'EOSQL' DROP DATABASE IF EXISTS migra_from; DROP DATABASE IF EXISTS migra_to; EOSQL } trap drop_migra_dbs 0 2 3 15 echo "Creating Migration from $FROM to $TO" # Install From into Database psql -q -X -1 -v ON_ERROR_STOP=1 -v CLIENT_MIN_MESSAGES=WARNING -f $FROMSQL $FROMDBURL >/dev/null || exit 1; # Install To into Database psql -q -X -1 -v ON_ERROR_STOP=1 -v CLIENT_MIN_MESSAGES=WARNING -f $TOSQL $TODBURL >/dev/null || exit 1; # Calculate the migration MIGRATION=$(mktemp) trap "rm $MIGRATION" 0 2 3 15 migra --schema pgstac --unsafe $FROMDBURL $TODBURL >$MIGRATION if [[ $DEBUG == 1 ]]; then echo "*************" cat $MIGRATION echo "*************" fi # Append wrapper around created migration with idempotent and transaction statements echo "SET client_min_messages TO WARNING;" >$MIGRATIONSQL echo "SET SEARCH_PATH to pgstac, public;" >>$MIGRATIONSQL cat $SQLDIR/000_idempotent_pre.sql >>$MIGRATIONSQL echo "-- BEGIN migra calculated SQL" >>$MIGRATIONSQL cat $MIGRATION >>$MIGRATIONSQL echo "-- END migra calculated SQL" >>$MIGRATIONSQL cat $SQLDIR/998_idempotent_post.sql >>$MIGRATIONSQL echo "SELECT set_version('${TO}');" >>$MIGRATIONSQL echo "Migration created at $MIGRATIONSQL." exit 0 ================================================ FILE: scripts/container-scripts/pgstac_restore ================================================ #!/bin/bash set -e function usage() { cat < Restore a PgSTAC pg_dump (-Fc) backup into a target database. pg_dump clears the search_path during restore, which breaks PgSTAC functions that reference PostGIS functions without schema qualification. This script works around that by installing a temporary event trigger on the target database that resets search_path before every DDL command, then running pg_restore directly (preserving its dependency ordering). Prerequisites on the target database: - Extensions: postgis, btree_gist, unaccent - Roles (optional): pgstac_admin, pgstac_read, pgstac_ingest (use --no-roles if roles don't exist; implies --no-owner --no-privileges) Options: -d, --dbname NAME Target database (default: \$PGDATABASE) -f, --file FILE Dump file path (also accepted as positional arg) --no-roles Skip role-dependent options (adds --no-owner --no-privileges) --create-extensions Create required extensions on target if missing -h, --help Show this help message Environment: Standard PG variables (PGHOST, PGPORT, PGUSER, PGPASSWORD, PGDATABASE) are respected for both psql and pg_restore. Examples: # Dump pg_dump -Fc --schema=pgstac -f pgstac_backup.dump mydb # Restore into an existing database with extensions already installed $(basename "$0") -d target_db pgstac_backup.dump # Restore, creating extensions and skipping ownership $(basename "$0") -d target_db --create-extensions --no-roles pgstac_backup.dump EOF exit "${1:-0}" } DBNAME="${PGDATABASE:-}" DUMPFILE="" NO_ROLES=0 CREATE_EXTENSIONS=0 while [[ $# -gt 0 ]]; do case "$1" in -d|--dbname) DBNAME="$2" shift 2 ;; -f|--file) DUMPFILE="$2" shift 2 ;; --no-roles) NO_ROLES=1 shift ;; --create-extensions) CREATE_EXTENSIONS=1 shift ;; -h|--help) usage 0 ;; -*) echo "Unknown option: $1" >&2 usage 1 ;; *) if [ -z "$DUMPFILE" ]; then DUMPFILE="$1" else echo "Unexpected argument: $1" >&2 usage 1 fi shift ;; esac done if [ -z "$DUMPFILE" ]; then echo "Error: dump file is required" >&2 usage 1 fi if [ ! -f "$DUMPFILE" ]; then echo "Error: dump file not found: $DUMPFILE" >&2 exit 1 fi if [ -z "$DBNAME" ]; then echo "Error: target database is required (-d or PGDATABASE)" >&2 usage 1 fi # Create extensions if requested if [ "$CREATE_EXTENSIONS" -eq 1 ]; then echo "Creating extensions on ${DBNAME} ..." psql -X -q -v ON_ERROR_STOP=1 -d "$DBNAME" </dev/null || true) if [ -z "$POSTGIS_SCHEMA" ]; then echo "Error: PostGIS extension not found on target database '${DBNAME}'." >&2 echo "Install PostGIS first or use --create-extensions." >&2 exit 1 fi # Build the search_path: pgstac + PostGIS schema + public (deduplicated) SEARCH_PATH="pgstac, ${POSTGIS_SCHEMA}" if [ "$POSTGIS_SCHEMA" != "public" ]; then SEARCH_PATH="${SEARCH_PATH}, public" fi echo "Using search_path: ${SEARCH_PATH}" # Build pg_restore flags RESTORE_FLAGS=() if [ "$NO_ROLES" -eq 1 ]; then RESTORE_FLAGS+=(--no-owner --no-privileges) fi echo "Restoring ${DUMPFILE} into ${DBNAME} ..." # pg_dump clears search_path during restore, which breaks PgSTAC functions # that reference PostGIS without schema qualification. We install a temporary # event trigger that resets search_path before every DDL command, then run # pg_restore directly. # Install temporary event trigger to fix search_path psql -X -v ON_ERROR_STOP=1 -d "$DBNAME" <&2 fi echo "Restore complete." ================================================ FILE: scripts/container-scripts/resetpgstac ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd ${PGSTAC_PGSTAC_DIR:-/opt/src/pgstac} set -e psql -v ON_ERROR_STOP=1 <<-EOSQL DROP SCHEMA IF EXISTS pgstac CASCADE; \i pgstac.sql SET SEARCH_PATH TO pgstac, public; \copy collections (content) FROM 'tests/testdata/collections.ndjson' \copy items_staging (content) FROM 'tests/testdata/items.ndjson' EOSQL ================================================ FILE: scripts/container-scripts/stageversion ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) SRCDIR=${PGSTAC_REPO_DIR:-/opt/src} cd $SRCDIR BASEDIR=$SRCDIR SQLDIR=$BASEDIR/pgstac/sql PYPGSTACDIR=$BASEDIR/pypgstac MIGRATIONSDIR=$BASEDIR/pgstac/migrations function usage() { cat <999_version.sql cat *.sql >$MIGRATIONSDIR/pgstac.${VERSION}.sql cd $BASEDIR/pgstac # make the base pgstac.sql a symbolic link to the most recent version rm pgstac.sql cp migrations/pgstac.${VERSION}.sql pgstac.sql # Update the version number in the appropriate places [[ $VERSION == 'unreleased' ]] && PYVERSION="${OLDVERSION}-dev" || PYVERSION="$VERSION" echo "Setting pypgstac version to $PYVERSION" cat < $PYPGSTACDIR/src/pypgstac/version.py """Version.""" __version__ = "${PYVERSION}" EOD sed -i "s/^version[ ]*=[ ]*.*$/version = \"${PYVERSION}\"/" $PYPGSTACDIR/pyproject.toml makemigration -f $OLDVERSION -t $VERSION ================================================ FILE: scripts/container-scripts/test ================================================ #!/bin/bash set -e SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) REPO_DIR=${PGSTAC_REPO_DIR:-/opt/src} export PATH="$SCRIPT_DIR:$PATH" if [[ -d "$REPO_DIR/pgstac" && -d "$REPO_DIR/pypgstac" ]]; then export SRCDIR="$REPO_DIR" elif [[ -d "$REPO_DIR/src/pgstac" && -d "$REPO_DIR/src/pypgstac" ]]; then export SRCDIR="$REPO_DIR/src" elif [[ -d "$SCRIPT_DIR/../../src/pgstac" && -d "$SCRIPT_DIR/../../src/pypgstac" ]]; then export SRCDIR="$(cd "$SCRIPT_DIR/../../src" && pwd)" else echo "Unable to find pgstac/pypgstac sources under '$REPO_DIR'." >&2 echo "Set PGSTAC_REPO_DIR to either a directory containing pgstac/ and pypgstac/ or a repository root containing src/pgstac and src/pypgstac." >&2 exit 1 fi export PGSTACDIR=$SRCDIR/pgstac if [[ "${CI}" ]]; then set -x fi function usage() { cat </dev/null 2>&1 || true psql -X -q -d postgres -c "ALTER DATABASE postgis REFRESH COLLATION VERSION;" >/dev/null 2>&1 || true } function test_formatting(){ cd $SRCDIR/pypgstac echo "Running ruff" ruff check src/pypgstac tests ruff format --check src/pypgstac tests echo "Running ty" ty check echo "Checking if there are any staged migrations." find $SRCDIR/pgstac/migrations | grep 'staged' && { echo "There are staged migrations in pypgstac/migrations. Please check migrations and remove staged suffix."; exit 1; } VERSION=$(python -c "from pypgstac.version import __version__; print(__version__)") echo $VERSION if echo $VERSION | grep "dev"; then VERSION="unreleased" fi echo "Checking whether base sql migration exists for pypgstac version." [ -f $SRCDIR/pgstac/migrations/pgstac."${VERSION}".sql ] || { echo "****FAIL No Migration exists pypgstac/migrations/pgstac.${VERSION}.sql"; exit 1; } echo "Congratulations! All formatting tests pass." } function test_pgtap(){ cd $PGSTACDIR TEMPLATEDB=${1:-pgstac_test_db_template} psql -X -q -v ON_ERROR_STOP=1 <"$TMPFILE" diff -Z -b -w -B --strip-trailing-cr --suppress-blank-empty -C 1 "$TMPFILE" $SQLOUTFILE && echo "TEST $SQLFILE PASSED" || { echo "***TEST FOR $SQLFILE FAILED***"; exit 1; } done psql -X -q -c "DROP DATABASE IF EXISTS pgstac_test_basicsql WITH (force);"; } function test_pypgstac(){ [[ $MESSAGELOG == 1 ]] && VERBOSE="-vvv" TEMPLATEDB=${1:-pgstac_test_db_template} cd $SRCDIR/pypgstac python -m venv venv source venv/bin/activate pip install --cache /tmp/.pipcache --upgrade pip pip install --cache /tmp/.pipcache -e . --no-deps psql -X -q -v ON_ERROR_STOP=1 </dev/null || stat -f%z "$DUMPFILE" 2>/dev/null) echo "Dump file size: ${DUMPSIZE} bytes" if [ "$DUMPSIZE" -lt 1000 ]; then echo "***FAIL: pg_dump produced suspiciously small file (${DUMPSIZE} bytes)***" exit 1 fi # Create target database and restore using pgstac_restore. # pgstac_restore detects PostGIS schema on the target, fixes the # search_path that pg_dump clears, and pipes through psql. psql -X -q -v ON_ERROR_STOP=1 < /dev/null && pwd ) cd $SCRIPT_DIR/.. function usage() { cat <&2 usage exit 1 ;; esac done $SCRIPT_DIR/runinpypgstac --build-policy "$BUILD_POLICY" --cpfiles format ================================================ FILE: scripts/makemigration ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. function usage() { cat < /dev/null && pwd ) cd $SCRIPT_DIR/.. function usage() { cat < /dev/null && pwd ) cd $SCRIPT_DIR/.. export PATH=$SCRIPT_DIR:$PATH set -e if [[ "${CI}" ]]; then set -x fi function first_available_pgport() { local port for port in 5439 5440 5441 5442 5443 5444 5445; do if ! ss -ltn "( sport = :$port )" 2>/dev/null | tail -n +2 | grep -q .; then echo "$port" return 0 fi done echo 5439 } function ensure_env_file() { if [[ ! -f .env && -f .env.example ]]; then cp .env.example .env local selected_port selected_port=$(first_available_pgport) if [[ "$selected_port" != "5439" ]]; then sed -i "s/^PGPORT=.*/PGPORT=${selected_port}/" .env fi echo "Created .env from .env.example" fi } ================================================ FILE: scripts/runinpypgstac ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. source "$SCRIPT_DIR/pgstacenv" set -e if [[ "${CI}" ]]; then set -x fi function usage() { cat < [command args...] Run a command inside the pypgstac development container. Options: --build Equivalent to --build-policy always. --build-policy POLICY One of: always, missing, never. --no-cache Rebuild images without Docker layer cache. --cpfiles Copy /opt/src back to the host after the command runs. -h, --help Show this help text. Examples: $(basename "$0") --build test --pypgstac $(basename "$0") --build-policy missing format $(basename "$0") --build-policy always --cpfiles stageversion 0.9.11 EOF } function ensure_images_exist() { docker image inspect pgstac >/dev/null 2>&1 && docker image inspect pypgstac >/dev/null 2>&1 } function wait_for_pgstac() { local attempts=60 local status="" local container_id="" while [[ $attempts -gt 0 ]]; do container_id=$(docker compose ps -q pgstac 2>/dev/null || true) status=$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$container_id" 2>/dev/null || true) if [[ "$status" == "healthy" || "$status" == "running" ]]; then return 0 fi attempts=$((attempts - 1)) sleep 1 done echo "Timed out waiting for pgstac to become ready." >&2 return 1 } [ "$#" -eq 0 ] && usage && exit 1 CONTAINER_ARGS=() BUILD_POLICY="${PGSTAC_BUILD_POLICY:-always}" while [[ $# -gt 0 ]]; do case "$1" in --build) BUILD_POLICY="always" shift ;; --build-policy) BUILD_POLICY="$2" shift 2 ;; --no-cache) NOCACHE="--no-cache" shift ;; --cpfiles) CPFILES=1 shift ;; -h|--help) usage exit 0 ;; --) shift break ;; *) break ;; esac done CONTAINER_ARGS=("$@") if [[ ${#CONTAINER_ARGS[@]} -eq 0 ]]; then usage exit 1 fi case "$BUILD_POLICY" in always|missing|never) ;; *) echo "Invalid build policy: $BUILD_POLICY" >&2 usage exit 1 ;; esac if [[ -n "$NOCACHE" && "$BUILD_POLICY" == "never" ]]; then echo "--no-cache cannot be used with --build-policy never." >&2 exit 1 fi if [[ "$BUILD_POLICY" == "always" || ( "$BUILD_POLICY" == "missing" && ! ensure_images_exist ) ]]; then echo "Building docker images..." ensure_env_file docker compose build ${NOCACHE} fi ensure_env_file PGSTAC_RUNNING=$(docker compose ps pgstac --status running -q) echo "PGSTAC_RUNNING=$PGSTAC_RUNNING" if [[ -z "$PGSTAC_RUNNING" ]]; then docker compose up -d pgstac wait_for_pgstac fi if [[ $CPFILES == 1 ]]; then echo "Running pypgstac worker" WORKER_ID=$(docker compose run -d --rm pypgstac tail -f /dev/null) echo "Executing ${CONTAINER_ARGS[@]} in pypgstac worker" docker exec "$WORKER_ID" "${CONTAINER_ARGS[@]}" echo "copying datafiles to host" docker cp "$WORKER_ID":/opt/src $SCRIPT_DIR/.. echo "killing pypgstac worker" docker kill "$WORKER_ID" >/dev/null else echo "Running ${CONTAINER_ARGS[@]} in pypgstacworker" docker compose run -T --rm pypgstac "${CONTAINER_ARGS[@]}" fi JOBEXITCODE=$? [[ $PGSTAC_RUNNING == "" ]] && docker compose stop pgstac exit $JOBEXITCODE ================================================ FILE: scripts/server ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. source "$SCRIPT_DIR/pgstacenv" set -e if [[ "${CI}" ]]; then set -x fi function usage() { if [[ -n "$1" ]]; then echo "$1" >&2 echo >&2 fi echo -n \ "Usage: $(basename "$0") [--detach] [--no-cache] [--help] Runs the development database. --detach: Run in detached mode. --no-cache: Rebuild container images from scratch before startup. " } DETACH_ARG="" while [[ "$#" > 0 ]]; do case $1 in --detach) DETACH_ARG="--detach" shift ;; --no-cache) NO_CACHE="--no-cache" shift ;; -h|--help) usage exit 0 ;; *) usage "Unknown option: $1" exit 1 ;; esac; done if [ "${BASH_SOURCE[0]}" = "${0}" ]; then ensure_env_file docker compose build ${NO_CACHE} docker compose up ${DETACH_ARG} $@ fi ================================================ FILE: scripts/setup ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. set -e if [[ "${CI}" ]]; then set -x fi function usage() { echo -n \ "Usage: $(basename "$0") [--help] Sets up this project for development. " } while [[ "$#" -gt 0 ]]; do case "$1" in -h|--help) usage exit 0 ;; *) echo "Unknown option: $1" >&2 usage exit 1 ;; esac done if [ "${BASH_SOURCE[0]}" = "${0}" ]; then # Build docker containers scripts/update echo "Bringing up database..." scripts/server --detach echo "Done." fi ================================================ FILE: scripts/stageversion ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. function usage() { cat < /dev/null && pwd ) function usage() { cat </dev/null || true) [[ -n "$changed" ]] } if [[ "$BUILD_POLICY" != "always" && $STRICT -eq 1 ]]; then if [[ $WATCH -eq 1 ]]; then echo "Watch mode requires --build-policy always unless --no-strict is set." >&2 exit 1 fi if has_rebuild_sensitive_changes; then echo "Refusing to run tests with build-policy '$BUILD_POLICY' while scripts/docker/src changes are present." >&2 echo "Use --build-policy always or --no-strict if you intentionally want to reuse an existing image." >&2 exit 1 fi fi RUN_ARGS=(--build-policy "$BUILD_POLICY" test) RUN_ARGS+=("${POSITIONAL[@]}") function run_once() { "$SCRIPT_DIR/runinpypgstac" "${RUN_ARGS[@]}" } if [[ $WATCH -eq 1 ]]; then if ! command -v inotifywait >/dev/null 2>&1; then if [[ $STRICT -eq 1 ]]; then echo "Watch mode requires inotifywait (from inotify-tools)." >&2 exit 1 fi echo "Watch mode requested but inotifywait is unavailable; running tests once because --no-strict was set." >&2 run_once exit $? fi while true; do run_once || true echo "Waiting for file changes..." inotifywait -qq -r -e close_write,create,delete,move "$SCRIPT_DIR/.."/scripts "$SCRIPT_DIR/.."/docker "$SCRIPT_DIR/.."/src "$SCRIPT_DIR/.."/.github "$SCRIPT_DIR/.."/CONTRIBUTING.md "$SCRIPT_DIR/.."/CLAUDE.md done fi run_once ================================================ FILE: scripts/update ================================================ #!/bin/bash SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) cd $SCRIPT_DIR/.. source "$SCRIPT_DIR/pgstacenv" set -e if [[ "${CI}" ]]; then set -x fi function usage() { echo -n \ "Usage: $(basename "$0") [--no-cache] Builds the docker containers for this project. --no-cache: Rebuild all containers from scratch. " } # Parse args NO_CACHE=""; while [[ "$#" > 0 ]]; do case $1 in --no-cache) NO_CACHE="--no-cache"; shift;; -h|--help) usage; exit 0;; *) echo "Unknown parameter passed: $1" >&2; usage; exit 1;; esac; done if [ "${BASH_SOURCE[0]}" = "${0}" ]; then echo "==Building images..." ensure_env_file docker compose build ${NO_CACHE} fi ================================================ FILE: src/pgstac/migrations/pgstac.0.1.9-0.2.3.sql ================================================ SET SEARCH_PATH to pgstac, public; INSERT INTO migrations (version) VALUES ('0.2.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.1.9.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS partman; CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT ARRAY(SELECT jsonb_array_elements_text(_js)); $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* converts a jsonb text array to comma delimited list of identifer quoted useful for constructing column lists for selects */ CREATE OR REPLACE FUNCTION array_idents(_js jsonb) RETURNS text AS $$ SELECT string_agg(quote_ident(v),',') FROM jsonb_array_elements_text(_js) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT (value->'properties'->>'datetime')::timestamptz; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(i, '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(e, '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP RAISE NOTICE 'path % val %', rec.path, rec.value; IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION properties_idx(_in jsonb) RETURNS jsonb AS $$ WITH t AS ( select array_to_string(path,'.') as path, lower(value::text)::jsonb as lowerval FROM jsonb_val_paths(_in) WHERE array_to_string(path,'.') not in ('datetime') ) SELECT jsonb_object_agg(path, lowerval) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; /* CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ BEGIN IF pg_trigger_depth() = 1 THEN PERFORM create_collection(NEW.content); RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger BEFORE INSERT ON collections FOR EACH ROW EXECUTE PROCEDURE collections_trigger_func(); */ SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS items ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED NOT NULL, geometry geometry GENERATED ALWAYS AS (stac_geom(content)) STORED NOT NULL, properties jsonb GENERATED ALWAYS as (properties_idx(content->'properties')) STORED, collection_id text GENERATED ALWAYS AS (content->>'collection') STORED NOT NULL, datetime timestamptz GENERATED ALWAYS AS (stac_datetime(content)) STORED NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (stac_datetime(content)) ; ALTER TABLE items ADD constraint items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) DEFERRABLE; CREATE TABLE items_template ( LIKE items ); ALTER TABLE items_template ADD PRIMARY KEY (id); /* CREATE TABLE IF NOT EXISTS items_search ( id text NOT NULL, geometry geometry NOT NULL, properties jsonb, collection_id text NOT NULL, datetime timestamptz NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE TABLE IF NOT EXISTS items_search_template ( LIKE items_search ) ; ALTER TABLE items_search_template ADD PRIMARY KEY (id); */ DELETE from partman.part_config WHERE parent_table = 'pgstac.items'; SELECT partman.create_parent( 'pgstac.items', 'datetime', 'native', 'weekly', p_template_table := 'pgstac.items_template', p_premake := 4 ); CREATE OR REPLACE FUNCTION make_partitions(st timestamptz, et timestamptz DEFAULT NULL) RETURNS BOOL AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', coalesce(et, st)), '1 week'::interval ) w ), w AS (SELECT array_agg(w) as w FROM t) SELECT CASE WHEN w IS NULL THEN NULL ELSE partman.create_partition_time('pgstac.items', w, true) END FROM w; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_partition(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE INDEX "datetime_id_idx" ON items (datetime, id); CREATE INDEX "properties_idx" ON items USING GIN (properties); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE TYPE item AS ( id text, geometry geometry, properties JSONB, collection_id text, datetime timestamptz ); /* Converts single feature into an items row */ /* CREATE OR REPLACE FUNCTION feature_to_item(value jsonb) RETURNS item AS $$ SELECT value->>'id' as id, CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry, properties_idx(value ->'properties') as properties, value->>'collection' as collection_id, (value->'properties'->>'datetime')::timestamptz as datetime ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; */ /* Takes a single feature, an array of features, or a feature collection and returns a set up individual items rows */ /* CREATE OR REPLACE FUNCTION features_to_items(value jsonb) RETURNS SETOF item AS $$ WITH features AS ( SELECT jsonb_array_elements( CASE WHEN jsonb_typeof(value) = 'array' THEN value WHEN value->>'type' = 'Feature' THEN '[]'::jsonb || value WHEN value->>'type' = 'FeatureCollection' THEN value->'features' ELSE NULL END ) as value ) SELECT feature_to_item(value) FROM features ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; */ CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ SELECT make_partitions(stac_datetime(data)); INSERT INTO items (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ DECLARE partition text; q text; newcontent jsonb; BEGIN PERFORM make_partitions(stac_datetime(data)); partition := get_partition(stac_datetime(data)); q := format($q$ INSERT INTO %I (content) VALUES ($1) ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content WHERE %I.content IS DISTINCT FROM EXCLUDED.content RETURNING content; $q$, partition, partition); EXECUTE q INTO newcontent USING (data); RAISE NOTICE 'newcontent: %', newcontent; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP RAISE NOTICE 'Analyzing %', p; EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; /* Trigger Function to cascade inserts/updates/deletes from items table to items_search table */ /* ALTER TABLE items_search ADD CONSTRAINT items_search_fk FOREIGN KEY (id) REFERENCES items(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION items_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN IF TG_OP = 'UPDATE' THEN RAISE NOTICE 'DELETING % BEFORE UPDATE', OLD; DELETE FROM items_search WHERE id = OLD.id AND datetime = (OLD.content->'properties'->>'datetime')::timestamptz; END IF; INSERT INTO items_search SELECT * FROM feature_to_item(NEW.content); RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_insert_trigger ON items; CREATE TRIGGER items_insert_trigger AFTER INSERT ON items FOR EACH ROW EXECUTE PROCEDURE items_trigger_func(); DROP TRIGGER IF EXISTS items_update_trigger ON items; CREATE TRIGGER items_update_trigger AFTER UPDATE ON items FOR EACH ROW WHEN (NEW.content IS DISTINCT FROM OLD.content) EXECUTE PROCEDURE items_trigger_func(); */ /* Trigger Function to cascade inserts/updates/deletes from items table to items_search table */ /* CREATE OR REPLACE FUNCTION items_search_trigger_delete_func() RETURNS TRIGGER AS $$ DECLARE BEGIN RAISE NOTICE 'Deleting from items_search: % Depth: %', OLD, pg_trigger_depth(); IF pg_trigger_depth()<3 THEN RAISE NOTICE 'DELETING WITH datetime'; DELETE FROM items_search WHERE id=OLD.id AND datetime=OLD.datetime; RETURN NULL; END IF; RETURN OLD; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_search_delete_trigger ON items_search; CREATE TRIGGER items_search_delete_trigger BEFORE DELETE ON items_search FOR EACH ROW EXECUTE PROCEDURE items_search_trigger_delete_func(); */ CREATE OR REPLACE FUNCTION backfill_partitions() RETURNS VOID AS $$ DECLARE BEGIN IF EXISTS (SELECT 1 FROM items_default LIMIT 1) THEN RAISE NOTICE 'Creating new partitions and moving data from default'; CREATE TEMP TABLE items_default_tmp ON COMMIT DROP AS SELECT datetime, content FROM items_default; TRUNCATE items_default; PERFORM make_partitions(min(datetime), max(datetime)) FROM items_default_tmp; INSERT INTO items (content) SELECT content FROM items_default_tmp; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION items_trigger_stmt_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_stmt_trigger ON items; CREATE TRIGGER items_stmt_trigger AFTER INSERT OR UPDATE OR DELETE ON items FOR EACH STATEMENT EXECUTE PROCEDURE items_trigger_stmt_func(); /* View to get a table of available items partitions with date ranges */ --DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE OR REPLACE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; --DROP VIEW IF EXISTS items_partitions; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS items_partitions; CREATE VIEW items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base WHERE est_cnt >0 ORDER BY 2 desc; CREATE OR REPLACE FUNCTION items_by_partition( IN _where text DEFAULT 'TRUE', IN _dtrange tstzrange DEFAULT tstzrange('-infinity','infinity'), IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE partition_query text; main_query text; batchcount int; counter int := 0; p record; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange ASC ; $q$); ELSE partition_query := format($q$ SELECT 'items' as partition WHERE $1 IS NOT NULL; $q$); END IF; RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query USING (_dtrange) LOOP IF lower(_dtrange)::timestamptz > '-infinity' THEN _where := concat(_where,format(' AND datetime >= %L',lower(_dtrange)::timestamptz::text)); END IF; IF upper(_dtrange)::timestamptz < 'infinity' THEN _where := concat(_where,format(' AND datetime <= %L',upper(_dtrange)::timestamptz::text)); END IF; main_query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s LIMIT %s - $1 $q$, p.partition::text, _where, _orderby, _limit ); RAISE NOTICE 'Partition Query %', main_query; RAISE NOTICE '%', counter; RETURN QUERY EXECUTE main_query USING counter; GET DIAGNOSTICS batchcount = ROW_COUNT; counter := counter + batchcount; RAISE NOTICE 'FOUND %', batchcount; IF counter >= _limit THEN EXIT; END IF; RAISE NOTICE 'ADDED % FOR A TOTAL OF %', batchcount, counter; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION split_stac_path(IN path text, OUT col text, OUT dotpath text, OUT jspath text, OUT jspathtext text) AS $$ WITH col AS ( SELECT CASE WHEN split_part(path, '.', 1) IN ('id', 'stac_version', 'stac_extensions','geometry','properties','assets','collection_id','datetime','links', 'extra_fields') THEN split_part(path, '.', 1) ELSE 'properties' END AS col ), dp AS ( SELECT col, ltrim(replace(path, col , ''),'.') as dotpath FROM col ), paths AS ( SELECT col, dotpath, regexp_split_to_table(dotpath,E'\\.') as path FROM dp ) SELECT col, btrim(concat(col,'.',dotpath),'.'), CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, regexp_replace( CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, E'>([^>]*)$','>>\1' ) FROM paths group by col, dotpath; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions for searching items */ CREATE OR REPLACE FUNCTION sort_base( IN _sort jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]', OUT key text, OUT col text, OUT dir text, OUT rdir text, OUT sort text, OUT rsort text ) RETURNS SETOF RECORD AS $$ WITH sorts AS ( SELECT value->>'field' as key, (split_stac_path(value->>'field')).jspathtext as col, coalesce(upper(value->>'direction'),'ASC') as dir FROM jsonb_array_elements('[]'::jsonb || coalesce(_sort,'[{"field":"datetime","direction":"desc"}]') ) ) SELECT key, col, dir, CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END as rdir, concat(col, ' ', dir, ' NULLS LAST ') AS sort, concat(col,' ', CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END, ' NULLS LAST ') AS rsort FROM sorts UNION ALL SELECT 'id', 'id', 'DESC', 'ASC', 'id DESC', 'id ASC' ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(sort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION rsort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(rsort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS box3d AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION in_array_q(col text, arr jsonb) RETURNS text AS $$ SELECT CASE jsonb_typeof(arr) WHEN 'array' THEN format('%I = ANY(textarr(%L))', col, arr) ELSE format('%I = %L', col, arr) END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION count_by_delim(text, text) RETURNS int AS $$ SELECT count(*) FROM regexp_split_to_table($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$ DECLARE ret text := ''; op text; jp text; att_parts RECORD; val_str text; BEGIN val_str := lower(jsonb_build_object('a',val)->>'a'); RAISE NOTICE 'val_str %', val_str; att_parts := split_stac_path(att); op := CASE _op WHEN 'eq' THEN '=' WHEN 'ge' THEN '>=' WHEN 'gt' THEN '>' WHEN 'le' THEN '<=' WHEN 'lt' THEN '<' WHEN 'ne' THEN '!=' WHEN 'neq' THEN '!=' WHEN 'startsWith' THEN 'LIKE' WHEN 'endsWith' THEN 'LIKE' WHEN 'contains' THEN 'LIKE' ELSE _op END; val_str := CASE _op WHEN 'startsWith' THEN concat(val_str, '%') WHEN 'endsWith' THEN concat('%', val_str) WHEN 'contains' THEN concat('%',val_str,'%') ELSE val_str END; RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.'); IF op = '=' AND att_parts.col = 'properties' --AND count_by_delim(att_parts.dotpath,'\.') = 2 THEN -- use jsonpath query to leverage index for eqaulity tests on single level deep properties jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb)); raise notice 'jp: %', jp; ret := format($q$ properties @? %L $q$, jp); ELSIF jsonb_typeof(val) = 'number' THEN ret := format('(%s)::numeric %s %s', att_parts.jspathtext, op, val); ELSE ret := format('%s %s %L', att_parts.jspathtext, op, val_str); END IF; RAISE NOTICE 'Op Query: %', ret; return ret; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION stac_query(_query jsonb) RETURNS TEXT[] AS $$ DECLARE qa text[]; att text; ops jsonb; op text; val jsonb; BEGIN FOR att, ops IN SELECT key, value FROM jsonb_each(_query) LOOP FOR op, val IN SELECT key, value FROM jsonb_each(ops) LOOP qa := array_append(qa, stac_query_op(att,op, val)); RAISE NOTICE '% % %', att, op, val; END LOOP; END LOOP; RETURN qa; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION filter_by_order(item_id text, _sort jsonb, _type text) RETURNS text AS $$ DECLARE item item; BEGIN SELECT * INTO item FROM items WHERE id=item_id; RETURN filter_by_order(item, _sort, _type); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; -- Used to create filters used for paging using the items id from the token CREATE OR REPLACE FUNCTION filter_by_order(_item item, _sort jsonb, _type text) RETURNS text AS $$ DECLARE sorts RECORD; filts text[]; itemval text; op text; idop text; ret text; eq_flag text; _item_j jsonb := to_jsonb(_item); BEGIN FOR sorts IN SELECT * FROM sort_base(_sort) LOOP IF sorts.col = 'datetime' THEN CONTINUE; END IF; IF sorts.col='id' AND _type IN ('prev','next') THEN eq_flag := ''; ELSE eq_flag := '='; END IF; op := concat( CASE WHEN _type in ('prev','first') AND sorts.dir = 'ASC' THEN '<' WHEN _type in ('last','next') AND sorts.dir = 'ASC' THEN '>' WHEN _type in ('prev','first') AND sorts.dir = 'DESC' THEN '>' WHEN _type in ('last','next') AND sorts.dir = 'DESC' THEN '<' END, eq_flag ); IF _item_j ? sorts.col THEN filts = array_append(filts, format('%s %s %L', sorts.col, op, _item_j->>sorts.col)); END IF; END LOOP; ret := coalesce(array_to_string(filts,' AND '), 'TRUE'); RAISE NOTICE 'Order Filter %', ret; RETURN ret; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION search_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ WITH t AS ( SELECT i, row_number() over () as r FROM jsonb_array_elements(j) i ), o AS ( SELECT i FROM t ORDER BY r DESC ) SELECT jsonb_agg(i) from o ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS SETOF jsonb AS $$ DECLARE qstart timestamptz := clock_timestamp(); _sort text := ''; _rsort text := ''; _limit int := 10; _geom geometry; qa text[]; pq text[]; query text; pq_prop record; pq_op record; prev_id text := NULL; next_id text := NULL; whereq text := 'TRUE'; links jsonb := '[]'::jsonb; token text; tok_val text; tok_q text := 'TRUE'; tok_sort text; first_id text; first_dt timestamptz; last_id text; sort text; rsort text; dt text[]; dqa text[]; dq text; mq_where text; startdt timestamptz; enddt timestamptz; item items%ROWTYPE; counter int := 0; batchcount int; month timestamptz; m record; _dtrange tstzrange := tstzrange('-infinity','infinity'); _dtsort text; _token_dtrange tstzrange := tstzrange('-infinity','infinity'); _token_record items%ROWTYPE; is_prev boolean := false; includes text[]; excludes text[]; BEGIN -- Create table from sort query of items to sort CREATE TEMP TABLE pgstac_tmp_sorts ON COMMIT DROP AS SELECT * FROM sort_base(_search->'sortby'); -- Get the datetime sort direction, necessary for efficient cycling through partitions SELECT INTO _dtsort dir FROM pgstac_tmp_sorts WHERE key='datetime'; RAISE NOTICE '_dtsort: %',_dtsort; SELECT INTO _sort string_agg(s.sort,', ') FROM pgstac_tmp_sorts s; SELECT INTO _rsort string_agg(s.rsort,', ') FROM pgstac_tmp_sorts s; tok_sort := _sort; -- Get datetime from query as a tstzrange IF _search ? 'datetime' THEN _dtrange := search_dtrange(_search->'datetime'); _token_dtrange := _dtrange; END IF; -- Get the paging token IF _search ? 'token' THEN token := _search->>'token'; tok_val := substr(token,6); IF starts_with(token, 'prev:') THEN is_prev := true; END IF; SELECT INTO _token_record * FROM items WHERE id=tok_val; IF (is_prev AND _dtsort = 'DESC') OR (not is_prev AND _dtsort = 'ASC') THEN _token_dtrange := tstzrange(_token_record.datetime, 'infinity'); ELSIF _dtsort IS NOT NULL THEN _token_dtrange := tstzrange('-infinity',_token_record.datetime); END IF; IF is_prev THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'first'); _sort := _rsort; ELSIF starts_with(token, 'next:') THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'last'); END IF; END IF; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'tok_q: % _token_dtrange: %', tok_q, _token_dtrange; IF _search ? 'ids' THEN RAISE NOTICE 'searching solely based on ids... %',_search; qa := array_append(qa, in_array_q('id', _search->'ids')); ELSE IF _search ? 'intersects' THEN _geom := ST_SetSRID(ST_GeomFromGeoJSON(_search->>'intersects'), 4326); ELSIF _search ? 'bbox' THEN _geom := bbox_geom(_search->'bbox'); END IF; IF _geom IS NOT NULL THEN qa := array_append(qa, format('st_intersects(geometry, %L::geometry)',_geom)); END IF; IF _search ? 'collections' THEN qa := array_append(qa, in_array_q('collection_id', _search->'collections')); END IF; IF _search ? 'query' THEN qa := array_cat(qa, stac_query(_search->'query') ); END IF; END IF; IF _search ? 'limit' THEN _limit := (_search->>'limit')::int; END IF; IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; RAISE NOTICE 'Includes: %, Excludes: %', includes, excludes; END IF; whereq := COALESCE(array_to_string(qa,' AND '),' TRUE '); dq := COALESCE(array_to_string(dqa,' AND '),' TRUE '); RAISE NOTICE 'timing before temp table: %', age(clock_timestamp(), qstart); CREATE TEMP TABLE results_page ON COMMIT DROP AS SELECT * FROM items_by_partition( concat(whereq, ' AND ', tok_q), _token_dtrange, _sort, _limit + 1 ); RAISE NOTICE 'timing after temp table: %', age(clock_timestamp(), qstart); RAISE NOTICE 'timing before min/max: %', age(clock_timestamp(), qstart); IF is_prev THEN SELECT INTO last_id, first_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; ELSE SELECT INTO first_id, last_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; END IF; RAISE NOTICE 'firstid: %, lastid %', first_id, last_id; RAISE NOTICE 'timing after min/max: %', age(clock_timestamp(), qstart); IF counter > _limit THEN next_id := last_id; RAISE NOTICE 'next_id: %', next_id; ELSE RAISE NOTICE 'No more next'; END IF; IF tok_q = 'TRUE' THEN RAISE NOTICE 'Not a paging query, no previous item'; ELSE RAISE NOTICE 'Getting previous item id'; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); SELECT INTO _token_record * FROM items WHERE id=first_id; IF _dtsort = 'DESC' THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSE _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; RAISE NOTICE '% %', _token_dtrange, _dtrange; SELECT id INTO prev_id FROM items_by_partition( concat(whereq, ' AND ', filter_by_order(first_id, _search->'sortby', 'prev')), _token_dtrange, _rsort, 1 ); RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'prev_id: %', prev_id; END IF; RETURN QUERY WITH features AS ( SELECT filter_jsonb(content, includes, excludes) as content FROM results_page LIMIT _limit ), j AS (SELECT jsonb_agg(content) as feature_arr FROM features) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce ( CASE WHEN is_prev THEN flip_jsonb_array(feature_arr) ELSE feature_arr END ,'[]'::jsonb), 'links', links, 'timeStamp', now(), 'next', next_id, 'prev', prev_id ) FROM j ; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; INSERT INTO migrations (version) VALUES ('0.1.9'); ================================================ FILE: src/pgstac/migrations/pgstac.0.2.3-0.2.4.sql ================================================ SET SEARCH_PATH to pgstac, public; INSERT INTO migrations (version) VALUES ('0.2.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.2.3.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS partman; CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT ARRAY(SELECT jsonb_array_elements_text(_js)); $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* converts a jsonb text array to comma delimited list of identifer quoted useful for constructing column lists for selects */ CREATE OR REPLACE FUNCTION array_idents(_js jsonb) RETURNS text AS $$ SELECT string_agg(quote_ident(v),',') FROM jsonb_array_elements_text(_js) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT (value->'properties'->>'datetime')::timestamptz; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(i, '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(e, '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP RAISE NOTICE 'path % val %', rec.path, rec.value; IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION properties_idx(_in jsonb) RETURNS jsonb AS $$ WITH t AS ( select array_to_string(path,'.') as path, lower(value::text)::jsonb as lowerval FROM jsonb_val_paths(_in) WHERE array_to_string(path,'.') not in ('datetime') ) SELECT jsonb_object_agg(path, lowerval) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; /* CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ BEGIN IF pg_trigger_depth() = 1 THEN PERFORM create_collection(NEW.content); RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger BEFORE INSERT ON collections FOR EACH ROW EXECUTE PROCEDURE collections_trigger_func(); */ SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS items ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED NOT NULL, geometry geometry GENERATED ALWAYS AS (stac_geom(content)) STORED NOT NULL, properties jsonb GENERATED ALWAYS as (properties_idx(content->'properties')) STORED, collection_id text GENERATED ALWAYS AS (content->>'collection') STORED NOT NULL, datetime timestamptz GENERATED ALWAYS AS (stac_datetime(content)) STORED NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (stac_datetime(content)) ; ALTER TABLE items ADD constraint items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) DEFERRABLE; CREATE TABLE items_template ( LIKE items ); ALTER TABLE items_template ADD PRIMARY KEY (id); /* CREATE TABLE IF NOT EXISTS items_search ( id text NOT NULL, geometry geometry NOT NULL, properties jsonb, collection_id text NOT NULL, datetime timestamptz NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE TABLE IF NOT EXISTS items_search_template ( LIKE items_search ) ; ALTER TABLE items_search_template ADD PRIMARY KEY (id); */ DELETE from partman.part_config WHERE parent_table = 'pgstac.items'; SELECT partman.create_parent( 'pgstac.items', 'datetime', 'native', 'weekly', p_template_table := 'pgstac.items_template', p_premake := 4 ); CREATE OR REPLACE FUNCTION make_partitions(st timestamptz, et timestamptz DEFAULT NULL) RETURNS BOOL AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', coalesce(et, st)), '1 week'::interval ) w ), w AS (SELECT array_agg(w) as w FROM t) SELECT CASE WHEN w IS NULL THEN NULL ELSE partman.create_partition_time('pgstac.items', w, true) END FROM w; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_partition(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE INDEX "datetime_id_idx" ON items (datetime, id); CREATE INDEX "properties_idx" ON items USING GIN (properties); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE TYPE item AS ( id text, geometry geometry, properties JSONB, collection_id text, datetime timestamptz ); /* Converts single feature into an items row */ /* CREATE OR REPLACE FUNCTION feature_to_item(value jsonb) RETURNS item AS $$ SELECT value->>'id' as id, CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry, properties_idx(value ->'properties') as properties, value->>'collection' as collection_id, (value->'properties'->>'datetime')::timestamptz as datetime ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; */ /* Takes a single feature, an array of features, or a feature collection and returns a set up individual items rows */ /* CREATE OR REPLACE FUNCTION features_to_items(value jsonb) RETURNS SETOF item AS $$ WITH features AS ( SELECT jsonb_array_elements( CASE WHEN jsonb_typeof(value) = 'array' THEN value WHEN value->>'type' = 'Feature' THEN '[]'::jsonb || value WHEN value->>'type' = 'FeatureCollection' THEN value->'features' ELSE NULL END ) as value ) SELECT feature_to_item(value) FROM features ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; */ CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ SELECT make_partitions(stac_datetime(data)); INSERT INTO items (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ DECLARE partition text; q text; newcontent jsonb; BEGIN PERFORM make_partitions(stac_datetime(data)); partition := get_partition(stac_datetime(data)); q := format($q$ INSERT INTO %I (content) VALUES ($1) ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content WHERE %I.content IS DISTINCT FROM EXCLUDED.content RETURNING content; $q$, partition, partition); EXECUTE q INTO newcontent USING (data); RAISE NOTICE 'newcontent: %', newcontent; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP RAISE NOTICE 'Analyzing %', p; EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; /* Trigger Function to cascade inserts/updates/deletes from items table to items_search table */ /* ALTER TABLE items_search ADD CONSTRAINT items_search_fk FOREIGN KEY (id) REFERENCES items(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION items_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN IF TG_OP = 'UPDATE' THEN RAISE NOTICE 'DELETING % BEFORE UPDATE', OLD; DELETE FROM items_search WHERE id = OLD.id AND datetime = (OLD.content->'properties'->>'datetime')::timestamptz; END IF; INSERT INTO items_search SELECT * FROM feature_to_item(NEW.content); RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_insert_trigger ON items; CREATE TRIGGER items_insert_trigger AFTER INSERT ON items FOR EACH ROW EXECUTE PROCEDURE items_trigger_func(); DROP TRIGGER IF EXISTS items_update_trigger ON items; CREATE TRIGGER items_update_trigger AFTER UPDATE ON items FOR EACH ROW WHEN (NEW.content IS DISTINCT FROM OLD.content) EXECUTE PROCEDURE items_trigger_func(); */ /* Trigger Function to cascade inserts/updates/deletes from items table to items_search table */ /* CREATE OR REPLACE FUNCTION items_search_trigger_delete_func() RETURNS TRIGGER AS $$ DECLARE BEGIN RAISE NOTICE 'Deleting from items_search: % Depth: %', OLD, pg_trigger_depth(); IF pg_trigger_depth()<3 THEN RAISE NOTICE 'DELETING WITH datetime'; DELETE FROM items_search WHERE id=OLD.id AND datetime=OLD.datetime; RETURN NULL; END IF; RETURN OLD; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_search_delete_trigger ON items_search; CREATE TRIGGER items_search_delete_trigger BEFORE DELETE ON items_search FOR EACH ROW EXECUTE PROCEDURE items_search_trigger_delete_func(); */ CREATE OR REPLACE FUNCTION backfill_partitions() RETURNS VOID AS $$ DECLARE BEGIN IF EXISTS (SELECT 1 FROM items_default LIMIT 1) THEN RAISE NOTICE 'Creating new partitions and moving data from default'; CREATE TEMP TABLE items_default_tmp ON COMMIT DROP AS SELECT datetime, content FROM items_default; TRUNCATE items_default; PERFORM make_partitions(min(datetime), max(datetime)) FROM items_default_tmp; INSERT INTO items (content) SELECT content FROM items_default_tmp; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION items_trigger_stmt_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_stmt_trigger ON items; CREATE TRIGGER items_stmt_trigger AFTER INSERT OR UPDATE OR DELETE ON items FOR EACH STATEMENT EXECUTE PROCEDURE items_trigger_stmt_func(); /* View to get a table of available items partitions with date ranges */ --DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE OR REPLACE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; --DROP VIEW IF EXISTS items_partitions; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS items_partitions; CREATE VIEW items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base WHERE est_cnt >0 ORDER BY 2 desc; CREATE OR REPLACE FUNCTION items_by_partition( IN _where text DEFAULT 'TRUE', IN _dtrange tstzrange DEFAULT tstzrange('-infinity','infinity'), IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE partition_query text; main_query text; batchcount int; counter int := 0; p record; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange ASC ; $q$); ELSE partition_query := format($q$ SELECT 'items' as partition WHERE $1 IS NOT NULL; $q$); END IF; RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query USING (_dtrange) LOOP IF lower(_dtrange)::timestamptz > '-infinity' THEN _where := concat(_where,format(' AND datetime >= %L',lower(_dtrange)::timestamptz::text)); END IF; IF upper(_dtrange)::timestamptz < 'infinity' THEN _where := concat(_where,format(' AND datetime <= %L',upper(_dtrange)::timestamptz::text)); END IF; main_query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s LIMIT %s - $1 $q$, p.partition::text, _where, _orderby, _limit ); RAISE NOTICE 'Partition Query %', main_query; RAISE NOTICE '%', counter; RETURN QUERY EXECUTE main_query USING counter; GET DIAGNOSTICS batchcount = ROW_COUNT; counter := counter + batchcount; RAISE NOTICE 'FOUND %', batchcount; IF counter >= _limit THEN EXIT; END IF; RAISE NOTICE 'ADDED % FOR A TOTAL OF %', batchcount, counter; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION split_stac_path(IN path text, OUT col text, OUT dotpath text, OUT jspath text, OUT jspathtext text) AS $$ WITH col AS ( SELECT CASE WHEN split_part(path, '.', 1) IN ('id', 'stac_version', 'stac_extensions','geometry','properties','assets','collection_id','datetime','links', 'extra_fields') THEN split_part(path, '.', 1) ELSE 'properties' END AS col ), dp AS ( SELECT col, ltrim(replace(path, col , ''),'.') as dotpath FROM col ), paths AS ( SELECT col, dotpath, regexp_split_to_table(dotpath,E'\\.') as path FROM dp ) SELECT col, btrim(concat(col,'.',dotpath),'.'), CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, regexp_replace( CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, E'>([^>]*)$','>>\1' ) FROM paths group by col, dotpath; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions for searching items */ CREATE OR REPLACE FUNCTION sort_base( IN _sort jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]', OUT key text, OUT col text, OUT dir text, OUT rdir text, OUT sort text, OUT rsort text ) RETURNS SETOF RECORD AS $$ WITH sorts AS ( SELECT value->>'field' as key, (split_stac_path(value->>'field')).jspathtext as col, coalesce(upper(value->>'direction'),'ASC') as dir FROM jsonb_array_elements('[]'::jsonb || coalesce(_sort,'[{"field":"datetime","direction":"desc"}]') ) ) SELECT key, col, dir, CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END as rdir, concat(col, ' ', dir, ' NULLS LAST ') AS sort, concat(col,' ', CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END, ' NULLS LAST ') AS rsort FROM sorts UNION ALL SELECT 'id', 'id', 'DESC', 'ASC', 'id DESC', 'id ASC' ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(sort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION rsort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(rsort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS box3d AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION in_array_q(col text, arr jsonb) RETURNS text AS $$ SELECT CASE jsonb_typeof(arr) WHEN 'array' THEN format('%I = ANY(textarr(%L))', col, arr) ELSE format('%I = %L', col, arr) END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION count_by_delim(text, text) RETURNS int AS $$ SELECT count(*) FROM regexp_split_to_table($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$ DECLARE ret text := ''; op text; jp text; att_parts RECORD; val_str text; BEGIN val_str := lower(jsonb_build_object('a',val)->>'a'); RAISE NOTICE 'val_str %', val_str; att_parts := split_stac_path(att); op := CASE _op WHEN 'eq' THEN '=' WHEN 'ge' THEN '>=' WHEN 'gt' THEN '>' WHEN 'le' THEN '<=' WHEN 'lt' THEN '<' WHEN 'ne' THEN '!=' WHEN 'neq' THEN '!=' WHEN 'startsWith' THEN 'LIKE' WHEN 'endsWith' THEN 'LIKE' WHEN 'contains' THEN 'LIKE' ELSE _op END; val_str := CASE _op WHEN 'startsWith' THEN concat(val_str, '%') WHEN 'endsWith' THEN concat('%', val_str) WHEN 'contains' THEN concat('%',val_str,'%') ELSE val_str END; RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.'); IF op = '=' AND att_parts.col = 'properties' --AND count_by_delim(att_parts.dotpath,'\.') = 2 THEN -- use jsonpath query to leverage index for eqaulity tests on single level deep properties jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb)); raise notice 'jp: %', jp; ret := format($q$ properties @? %L $q$, jp); ELSIF jsonb_typeof(val) = 'number' THEN ret := format('(%s)::numeric %s %s', att_parts.jspathtext, op, val); ELSE ret := format('%s %s %L', att_parts.jspathtext, op, val_str); END IF; RAISE NOTICE 'Op Query: %', ret; return ret; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION stac_query(_query jsonb) RETURNS TEXT[] AS $$ DECLARE qa text[]; att text; ops jsonb; op text; val jsonb; BEGIN FOR att, ops IN SELECT key, value FROM jsonb_each(_query) LOOP FOR op, val IN SELECT key, value FROM jsonb_each(ops) LOOP qa := array_append(qa, stac_query_op(att,op, val)); RAISE NOTICE '% % %', att, op, val; END LOOP; END LOOP; RETURN qa; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION filter_by_order(item_id text, _sort jsonb, _type text) RETURNS text AS $$ DECLARE item item; BEGIN SELECT * INTO item FROM items WHERE id=item_id; RETURN filter_by_order(item, _sort, _type); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; -- Used to create filters used for paging using the items id from the token CREATE OR REPLACE FUNCTION filter_by_order(_item item, _sort jsonb, _type text) RETURNS text AS $$ DECLARE sorts RECORD; filts text[]; itemval text; op text; idop text; ret text; eq_flag text; _item_j jsonb := to_jsonb(_item); BEGIN FOR sorts IN SELECT * FROM sort_base(_sort) LOOP IF sorts.col = 'datetime' THEN CONTINUE; END IF; IF sorts.col='id' AND _type IN ('prev','next') THEN eq_flag := ''; ELSE eq_flag := '='; END IF; op := concat( CASE WHEN _type in ('prev','first') AND sorts.dir = 'ASC' THEN '<' WHEN _type in ('last','next') AND sorts.dir = 'ASC' THEN '>' WHEN _type in ('prev','first') AND sorts.dir = 'DESC' THEN '>' WHEN _type in ('last','next') AND sorts.dir = 'DESC' THEN '<' END, eq_flag ); IF _item_j ? sorts.col THEN filts = array_append(filts, format('%s %s %L', sorts.col, op, _item_j->>sorts.col)); END IF; END LOOP; ret := coalesce(array_to_string(filts,' AND '), 'TRUE'); RAISE NOTICE 'Order Filter %', ret; RETURN ret; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION search_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ WITH t AS ( SELECT i, row_number() over () as r FROM jsonb_array_elements(j) i ), o AS ( SELECT i FROM t ORDER BY r DESC ) SELECT jsonb_agg(i) from o ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS SETOF jsonb AS $$ DECLARE qstart timestamptz := clock_timestamp(); _sort text := ''; _rsort text := ''; _limit int := 10; _geom geometry; qa text[]; pq text[]; query text; pq_prop record; pq_op record; prev_id text := NULL; next_id text := NULL; whereq text := 'TRUE'; links jsonb := '[]'::jsonb; token text; tok_val text; tok_q text := 'TRUE'; tok_sort text; first_id text; first_dt timestamptz; last_id text; sort text; rsort text; dt text[]; dqa text[]; dq text; mq_where text; startdt timestamptz; enddt timestamptz; item items%ROWTYPE; counter int := 0; batchcount int; month timestamptz; m record; _dtrange tstzrange := tstzrange('-infinity','infinity'); _dtsort text; _token_dtrange tstzrange := tstzrange('-infinity','infinity'); _token_record items%ROWTYPE; is_prev boolean := false; includes text[]; excludes text[]; BEGIN -- Create table from sort query of items to sort CREATE TEMP TABLE pgstac_tmp_sorts ON COMMIT DROP AS SELECT * FROM sort_base(_search->'sortby'); -- Get the datetime sort direction, necessary for efficient cycling through partitions SELECT INTO _dtsort dir FROM pgstac_tmp_sorts WHERE key='datetime'; RAISE NOTICE '_dtsort: %',_dtsort; SELECT INTO _sort string_agg(s.sort,', ') FROM pgstac_tmp_sorts s; SELECT INTO _rsort string_agg(s.rsort,', ') FROM pgstac_tmp_sorts s; tok_sort := _sort; -- Get datetime from query as a tstzrange IF _search ? 'datetime' THEN _dtrange := search_dtrange(_search->'datetime'); _token_dtrange := _dtrange; END IF; -- Get the paging token IF _search ? 'token' THEN token := _search->>'token'; tok_val := substr(token,6); IF starts_with(token, 'prev:') THEN is_prev := true; END IF; SELECT INTO _token_record * FROM items WHERE id=tok_val; IF (is_prev AND _dtsort = 'DESC') OR (not is_prev AND _dtsort = 'ASC') THEN _token_dtrange := tstzrange(_token_record.datetime, 'infinity'); ELSIF _dtsort IS NOT NULL THEN _token_dtrange := tstzrange('-infinity',_token_record.datetime); END IF; IF is_prev THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'first'); _sort := _rsort; ELSIF starts_with(token, 'next:') THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'last'); END IF; END IF; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'tok_q: % _token_dtrange: %', tok_q, _token_dtrange; IF _search ? 'ids' THEN RAISE NOTICE 'searching solely based on ids... %',_search; qa := array_append(qa, in_array_q('id', _search->'ids')); ELSE IF _search ? 'intersects' THEN _geom := ST_SetSRID(ST_GeomFromGeoJSON(_search->>'intersects'), 4326); ELSIF _search ? 'bbox' THEN _geom := bbox_geom(_search->'bbox'); END IF; IF _geom IS NOT NULL THEN qa := array_append(qa, format('st_intersects(geometry, %L::geometry)',_geom)); END IF; IF _search ? 'collections' THEN qa := array_append(qa, in_array_q('collection_id', _search->'collections')); END IF; IF _search ? 'query' THEN qa := array_cat(qa, stac_query(_search->'query') ); END IF; END IF; IF _search ? 'limit' THEN _limit := (_search->>'limit')::int; END IF; IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; RAISE NOTICE 'Includes: %, Excludes: %', includes, excludes; END IF; whereq := COALESCE(array_to_string(qa,' AND '),' TRUE '); dq := COALESCE(array_to_string(dqa,' AND '),' TRUE '); RAISE NOTICE 'timing before temp table: %', age(clock_timestamp(), qstart); CREATE TEMP TABLE results_page ON COMMIT DROP AS SELECT * FROM items_by_partition( concat(whereq, ' AND ', tok_q), _token_dtrange, _sort, _limit + 1 ); RAISE NOTICE 'timing after temp table: %', age(clock_timestamp(), qstart); RAISE NOTICE 'timing before min/max: %', age(clock_timestamp(), qstart); IF is_prev THEN SELECT INTO last_id, first_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; ELSE SELECT INTO first_id, last_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; END IF; RAISE NOTICE 'firstid: %, lastid %', first_id, last_id; RAISE NOTICE 'timing after min/max: %', age(clock_timestamp(), qstart); IF counter > _limit THEN next_id := last_id; RAISE NOTICE 'next_id: %', next_id; ELSE RAISE NOTICE 'No more next'; END IF; IF tok_q = 'TRUE' THEN RAISE NOTICE 'Not a paging query, no previous item'; ELSE RAISE NOTICE 'Getting previous item id'; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); SELECT INTO _token_record * FROM items WHERE id=first_id; IF _dtsort = 'DESC' THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSE _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; RAISE NOTICE '% %', _token_dtrange, _dtrange; SELECT id INTO prev_id FROM items_by_partition( concat(whereq, ' AND ', filter_by_order(first_id, _search->'sortby', 'prev')), _token_dtrange, _rsort, 1 ); RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'prev_id: %', prev_id; END IF; RETURN QUERY WITH features AS ( SELECT filter_jsonb(content, includes, excludes) as content FROM results_page LIMIT _limit ), j AS (SELECT jsonb_agg(content) as feature_arr FROM features) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce ( CASE WHEN is_prev THEN flip_jsonb_array(feature_arr) ELSE feature_arr END ,'[]'::jsonb), 'links', links, 'timeStamp', now(), 'next', next_id, 'prev', prev_id ) FROM j ; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; INSERT INTO migrations (version) VALUES ('0.2.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.2.4-0.2.5.sql ================================================ SET SEARCH_PATH TO pgstac, public; BEGIN; DROP FUNCTION IF EXISTS bbox_geom; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS SETOF jsonb AS $$ DECLARE qstart timestamptz := clock_timestamp(); _sort text := ''; _rsort text := ''; _limit int := 10; _geom geometry; qa text[]; pq text[]; query text; pq_prop record; pq_op record; prev_id text := NULL; next_id text := NULL; whereq text := 'TRUE'; links jsonb := '[]'::jsonb; token text; tok_val text; tok_q text := 'TRUE'; tok_sort text; first_id text; first_dt timestamptz; last_id text; sort text; rsort text; dt text[]; dqa text[]; dq text; mq_where text; startdt timestamptz; enddt timestamptz; item items%ROWTYPE; counter int := 0; batchcount int; month timestamptz; m record; _dtrange tstzrange := tstzrange('-infinity','infinity'); _dtsort text; _token_dtrange tstzrange := tstzrange('-infinity','infinity'); _token_record items%ROWTYPE; is_prev boolean := false; includes text[]; excludes text[]; BEGIN -- Create table from sort query of items to sort CREATE TEMP TABLE pgstac_tmp_sorts ON COMMIT DROP AS SELECT * FROM sort_base(_search->'sortby'); -- Get the datetime sort direction, necessary for efficient cycling through partitions SELECT INTO _dtsort dir FROM pgstac_tmp_sorts WHERE key='datetime'; RAISE NOTICE '_dtsort: %',_dtsort; SELECT INTO _sort string_agg(s.sort,', ') FROM pgstac_tmp_sorts s; SELECT INTO _rsort string_agg(s.rsort,', ') FROM pgstac_tmp_sorts s; tok_sort := _sort; -- Get datetime from query as a tstzrange IF _search ? 'datetime' THEN _dtrange := search_dtrange(_search->'datetime'); _token_dtrange := _dtrange; END IF; -- Get the paging token IF _search ? 'token' THEN token := _search->>'token'; tok_val := substr(token,6); IF starts_with(token, 'prev:') THEN is_prev := true; END IF; SELECT INTO _token_record * FROM items WHERE id=tok_val; IF (is_prev AND _dtsort = 'DESC') OR (not is_prev AND _dtsort = 'ASC') THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSIF _dtsort IS NOT NULL THEN _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; IF is_prev THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'first'); _sort := _rsort; ELSIF starts_with(token, 'next:') THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'last'); END IF; END IF; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'tok_q: % _token_dtrange: %', tok_q, _token_dtrange; IF _search ? 'ids' THEN RAISE NOTICE 'searching solely based on ids... %',_search; qa := array_append(qa, in_array_q('id', _search->'ids')); ELSE IF _search ? 'intersects' THEN _geom := ST_SetSRID(ST_GeomFromGeoJSON(_search->>'intersects'), 4326); ELSIF _search ? 'bbox' THEN _geom := bbox_geom(_search->'bbox'); END IF; IF _geom IS NOT NULL THEN qa := array_append(qa, format('st_intersects(geometry, %L::geometry)',_geom)); END IF; IF _search ? 'collections' THEN qa := array_append(qa, in_array_q('collection_id', _search->'collections')); END IF; IF _search ? 'query' THEN qa := array_cat(qa, stac_query(_search->'query') ); END IF; END IF; IF _search ? 'limit' THEN _limit := (_search->>'limit')::int; END IF; IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; RAISE NOTICE 'Includes: %, Excludes: %', includes, excludes; END IF; whereq := COALESCE(array_to_string(qa,' AND '),' TRUE '); dq := COALESCE(array_to_string(dqa,' AND '),' TRUE '); RAISE NOTICE 'timing before temp table: %', age(clock_timestamp(), qstart); CREATE TEMP TABLE results_page ON COMMIT DROP AS SELECT * FROM items_by_partition( concat(whereq, ' AND ', tok_q), _token_dtrange, _sort, _limit + 1 ); RAISE NOTICE 'timing after temp table: %', age(clock_timestamp(), qstart); RAISE NOTICE 'timing before min/max: %', age(clock_timestamp(), qstart); IF is_prev THEN SELECT INTO last_id, first_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; ELSE SELECT INTO first_id, last_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; END IF; RAISE NOTICE 'firstid: %, lastid %', first_id, last_id; RAISE NOTICE 'timing after min/max: %', age(clock_timestamp(), qstart); IF counter > _limit THEN next_id := last_id; RAISE NOTICE 'next_id: %', next_id; ELSE RAISE NOTICE 'No more next'; END IF; IF tok_q = 'TRUE' THEN RAISE NOTICE 'Not a paging query, no previous item'; ELSE RAISE NOTICE 'Getting previous item id'; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); SELECT INTO _token_record * FROM items WHERE id=first_id; IF _dtsort = 'DESC' THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSE _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; RAISE NOTICE '% %', _token_dtrange, _dtrange; SELECT id INTO prev_id FROM items_by_partition( concat(whereq, ' AND ', filter_by_order(first_id, _search->'sortby', 'prev')), _token_dtrange, _rsort, 1 ); RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'prev_id: %', prev_id; END IF; RETURN QUERY WITH features AS ( SELECT filter_jsonb(content, includes, excludes) as content FROM results_page LIMIT _limit ), j AS (SELECT jsonb_agg(content) as feature_arr FROM features) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce ( CASE WHEN is_prev THEN flip_jsonb_array(feature_arr) ELSE feature_arr END ,'[]'::jsonb), 'links', links, 'timeStamp', now(), 'next', next_id, 'prev', prev_id ) FROM j ; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; INSERT INTO migrations (version) VALUES ('0.2.5'); COMMIT; ================================================ FILE: src/pgstac/migrations/pgstac.0.2.4-0.2.7.sql ================================================ SET SEARCH_PATH TO pgstac, public; BEGIN; CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$ DECLARE ret text := ''; op text; jp text; att_parts RECORD; val_str text; prop_path text; BEGIN val_str := lower(jsonb_build_object('a',val)->>'a'); RAISE NOTICE 'val_str %', val_str; att_parts := split_stac_path(att); prop_path := replace(att_parts.dotpath, 'properties.', ''); op := CASE _op WHEN 'eq' THEN '=' WHEN 'gte' THEN '>=' WHEN 'gt' THEN '>' WHEN 'lte' THEN '<=' WHEN 'lt' THEN '<' WHEN 'ne' THEN '!=' WHEN 'neq' THEN '!=' WHEN 'startsWith' THEN 'LIKE' WHEN 'endsWith' THEN 'LIKE' WHEN 'contains' THEN 'LIKE' ELSE _op END; val_str := CASE _op WHEN 'startsWith' THEN concat(val_str, '%') WHEN 'endsWith' THEN concat('%', val_str) WHEN 'contains' THEN concat('%',val_str,'%') ELSE val_str END; RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.'); IF op = '=' AND att_parts.col = 'properties' --AND count_by_delim(att_parts.dotpath,'\.') = 2 THEN -- use jsonpath query to leverage index for eqaulity tests on single level deep properties jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb)); raise notice 'jp: %', jp; ret := format($q$ properties @? %L $q$, jp); ELSIF jsonb_typeof(val) = 'number' THEN ret := format('properties ? %L AND (%s)::numeric %s %s', prop_path, att_parts.jspathtext, op, val); ELSE ret := format('properties ? %L AND %s %s %L', prop_path ,att_parts.jspathtext, op, val_str); END IF; RAISE NOTICE 'Op Query: %', ret; return ret; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS bbox_geom; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS SETOF jsonb AS $$ DECLARE qstart timestamptz := clock_timestamp(); _sort text := ''; _rsort text := ''; _limit int := 10; _geom geometry; qa text[]; pq text[]; query text; pq_prop record; pq_op record; prev_id text := NULL; next_id text := NULL; whereq text := 'TRUE'; links jsonb := '[]'::jsonb; token text; tok_val text; tok_q text := 'TRUE'; tok_sort text; first_id text; first_dt timestamptz; last_id text; sort text; rsort text; dt text[]; dqa text[]; dq text; mq_where text; startdt timestamptz; enddt timestamptz; item items%ROWTYPE; counter int := 0; batchcount int; month timestamptz; m record; _dtrange tstzrange := tstzrange('-infinity','infinity'); _dtsort text; _token_dtrange tstzrange := tstzrange('-infinity','infinity'); _token_record items%ROWTYPE; is_prev boolean := false; includes text[]; excludes text[]; BEGIN -- Create table from sort query of items to sort CREATE TEMP TABLE pgstac_tmp_sorts ON COMMIT DROP AS SELECT * FROM sort_base(_search->'sortby'); -- Get the datetime sort direction, necessary for efficient cycling through partitions SELECT INTO _dtsort dir FROM pgstac_tmp_sorts WHERE key='datetime'; RAISE NOTICE '_dtsort: %',_dtsort; SELECT INTO _sort string_agg(s.sort,', ') FROM pgstac_tmp_sorts s; SELECT INTO _rsort string_agg(s.rsort,', ') FROM pgstac_tmp_sorts s; tok_sort := _sort; -- Get datetime from query as a tstzrange IF _search ? 'datetime' THEN _dtrange := search_dtrange(_search->'datetime'); _token_dtrange := _dtrange; END IF; -- Get the paging token IF _search ? 'token' THEN token := _search->>'token'; tok_val := substr(token,6); IF starts_with(token, 'prev:') THEN is_prev := true; END IF; SELECT INTO _token_record * FROM items WHERE id=tok_val; IF (is_prev AND _dtsort = 'DESC') OR (not is_prev AND _dtsort = 'ASC') THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSIF _dtsort IS NOT NULL THEN _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; IF is_prev THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'first'); _sort := _rsort; ELSIF starts_with(token, 'next:') THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'last'); END IF; END IF; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'tok_q: % _token_dtrange: %', tok_q, _token_dtrange; IF _search ? 'ids' THEN RAISE NOTICE 'searching solely based on ids... %',_search; qa := array_append(qa, in_array_q('id', _search->'ids')); ELSE IF _search ? 'intersects' THEN _geom := ST_SetSRID(ST_GeomFromGeoJSON(_search->>'intersects'), 4326); ELSIF _search ? 'bbox' THEN _geom := bbox_geom(_search->'bbox'); END IF; IF _geom IS NOT NULL THEN qa := array_append(qa, format('st_intersects(geometry, %L::geometry)',_geom)); END IF; IF _search ? 'collections' THEN qa := array_append(qa, in_array_q('collection_id', _search->'collections')); END IF; IF _search ? 'query' THEN qa := array_cat(qa, stac_query(_search->'query') ); END IF; END IF; IF _search ? 'limit' THEN _limit := (_search->>'limit')::int; END IF; IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; RAISE NOTICE 'Includes: %, Excludes: %', includes, excludes; END IF; whereq := COALESCE(array_to_string(qa,' AND '),' TRUE '); dq := COALESCE(array_to_string(dqa,' AND '),' TRUE '); RAISE NOTICE 'timing before temp table: %', age(clock_timestamp(), qstart); CREATE TEMP TABLE results_page ON COMMIT DROP AS SELECT * FROM items_by_partition( concat(whereq, ' AND ', tok_q), _token_dtrange, _sort, _limit + 1 ); RAISE NOTICE 'timing after temp table: %', age(clock_timestamp(), qstart); RAISE NOTICE 'timing before min/max: %', age(clock_timestamp(), qstart); IF is_prev THEN SELECT INTO last_id, first_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; ELSE SELECT INTO first_id, last_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; END IF; RAISE NOTICE 'firstid: %, lastid %', first_id, last_id; RAISE NOTICE 'timing after min/max: %', age(clock_timestamp(), qstart); IF counter > _limit THEN next_id := last_id; RAISE NOTICE 'next_id: %', next_id; ELSE RAISE NOTICE 'No more next'; END IF; IF tok_q = 'TRUE' THEN RAISE NOTICE 'Not a paging query, no previous item'; ELSE RAISE NOTICE 'Getting previous item id'; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); SELECT INTO _token_record * FROM items WHERE id=first_id; IF _dtsort = 'DESC' THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSE _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; RAISE NOTICE '% %', _token_dtrange, _dtrange; SELECT id INTO prev_id FROM items_by_partition( concat(whereq, ' AND ', filter_by_order(first_id, _search->'sortby', 'prev')), _token_dtrange, _rsort, 1 ); RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'prev_id: %', prev_id; END IF; RETURN QUERY WITH features AS ( SELECT filter_jsonb(content, includes, excludes) as content FROM results_page LIMIT _limit ), j AS (SELECT jsonb_agg(content) as feature_arr FROM features) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce ( CASE WHEN is_prev THEN flip_jsonb_array(feature_arr) ELSE feature_arr END ,'[]'::jsonb), 'links', links, 'timeStamp', now(), 'next', next_id, 'prev', prev_id ) FROM j ; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; INSERT INTO migrations (version) VALUES ('0.2.7'); COMMIT; ================================================ FILE: src/pgstac/migrations/pgstac.0.2.4.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS partman; CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT ARRAY(SELECT jsonb_array_elements_text(_js)); $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* converts a jsonb text array to comma delimited list of identifer quoted useful for constructing column lists for selects */ CREATE OR REPLACE FUNCTION array_idents(_js jsonb) RETURNS text AS $$ SELECT string_agg(quote_ident(v),',') FROM jsonb_array_elements_text(_js) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT (value->'properties'->>'datetime')::timestamptz; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(i, '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(e, '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP RAISE NOTICE 'path % val %', rec.path, rec.value; IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION properties_idx(_in jsonb) RETURNS jsonb AS $$ WITH t AS ( select array_to_string(path,'.') as path, lower(value::text)::jsonb as lowerval FROM jsonb_val_paths(_in) WHERE array_to_string(path,'.') not in ('datetime') ) SELECT jsonb_object_agg(path, lowerval) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; /* CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ BEGIN IF pg_trigger_depth() = 1 THEN PERFORM create_collection(NEW.content); RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger BEFORE INSERT ON collections FOR EACH ROW EXECUTE PROCEDURE collections_trigger_func(); */ SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS items ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED NOT NULL, geometry geometry GENERATED ALWAYS AS (stac_geom(content)) STORED NOT NULL, properties jsonb GENERATED ALWAYS as (properties_idx(content->'properties')) STORED, collection_id text GENERATED ALWAYS AS (content->>'collection') STORED NOT NULL, datetime timestamptz GENERATED ALWAYS AS (stac_datetime(content)) STORED NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (stac_datetime(content)) ; ALTER TABLE items ADD constraint items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) DEFERRABLE; CREATE TABLE items_template ( LIKE items ); ALTER TABLE items_template ADD PRIMARY KEY (id); /* CREATE TABLE IF NOT EXISTS items_search ( id text NOT NULL, geometry geometry NOT NULL, properties jsonb, collection_id text NOT NULL, datetime timestamptz NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE TABLE IF NOT EXISTS items_search_template ( LIKE items_search ) ; ALTER TABLE items_search_template ADD PRIMARY KEY (id); */ DELETE from partman.part_config WHERE parent_table = 'pgstac.items'; SELECT partman.create_parent( 'pgstac.items', 'datetime', 'native', 'weekly', p_template_table := 'pgstac.items_template', p_premake := 4 ); CREATE OR REPLACE FUNCTION make_partitions(st timestamptz, et timestamptz DEFAULT NULL) RETURNS BOOL AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', coalesce(et, st)), '1 week'::interval ) w ), w AS (SELECT array_agg(w) as w FROM t) SELECT CASE WHEN w IS NULL THEN NULL ELSE partman.create_partition_time('pgstac.items', w, true) END FROM w; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_partition(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE INDEX "datetime_id_idx" ON items (datetime, id); CREATE INDEX "properties_idx" ON items USING GIN (properties); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE TYPE item AS ( id text, geometry geometry, properties JSONB, collection_id text, datetime timestamptz ); /* Converts single feature into an items row */ /* CREATE OR REPLACE FUNCTION feature_to_item(value jsonb) RETURNS item AS $$ SELECT value->>'id' as id, CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry, properties_idx(value ->'properties') as properties, value->>'collection' as collection_id, (value->'properties'->>'datetime')::timestamptz as datetime ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; */ /* Takes a single feature, an array of features, or a feature collection and returns a set up individual items rows */ /* CREATE OR REPLACE FUNCTION features_to_items(value jsonb) RETURNS SETOF item AS $$ WITH features AS ( SELECT jsonb_array_elements( CASE WHEN jsonb_typeof(value) = 'array' THEN value WHEN value->>'type' = 'Feature' THEN '[]'::jsonb || value WHEN value->>'type' = 'FeatureCollection' THEN value->'features' ELSE NULL END ) as value ) SELECT feature_to_item(value) FROM features ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; */ CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ SELECT make_partitions(stac_datetime(data)); INSERT INTO items (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ DECLARE partition text; q text; newcontent jsonb; BEGIN PERFORM make_partitions(stac_datetime(data)); partition := get_partition(stac_datetime(data)); q := format($q$ INSERT INTO %I (content) VALUES ($1) ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content WHERE %I.content IS DISTINCT FROM EXCLUDED.content RETURNING content; $q$, partition, partition); EXECUTE q INTO newcontent USING (data); RAISE NOTICE 'newcontent: %', newcontent; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP RAISE NOTICE 'Analyzing %', p; EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; /* Trigger Function to cascade inserts/updates/deletes from items table to items_search table */ /* ALTER TABLE items_search ADD CONSTRAINT items_search_fk FOREIGN KEY (id) REFERENCES items(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION items_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN IF TG_OP = 'UPDATE' THEN RAISE NOTICE 'DELETING % BEFORE UPDATE', OLD; DELETE FROM items_search WHERE id = OLD.id AND datetime = (OLD.content->'properties'->>'datetime')::timestamptz; END IF; INSERT INTO items_search SELECT * FROM feature_to_item(NEW.content); RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_insert_trigger ON items; CREATE TRIGGER items_insert_trigger AFTER INSERT ON items FOR EACH ROW EXECUTE PROCEDURE items_trigger_func(); DROP TRIGGER IF EXISTS items_update_trigger ON items; CREATE TRIGGER items_update_trigger AFTER UPDATE ON items FOR EACH ROW WHEN (NEW.content IS DISTINCT FROM OLD.content) EXECUTE PROCEDURE items_trigger_func(); */ /* Trigger Function to cascade inserts/updates/deletes from items table to items_search table */ /* CREATE OR REPLACE FUNCTION items_search_trigger_delete_func() RETURNS TRIGGER AS $$ DECLARE BEGIN RAISE NOTICE 'Deleting from items_search: % Depth: %', OLD, pg_trigger_depth(); IF pg_trigger_depth()<3 THEN RAISE NOTICE 'DELETING WITH datetime'; DELETE FROM items_search WHERE id=OLD.id AND datetime=OLD.datetime; RETURN NULL; END IF; RETURN OLD; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_search_delete_trigger ON items_search; CREATE TRIGGER items_search_delete_trigger BEFORE DELETE ON items_search FOR EACH ROW EXECUTE PROCEDURE items_search_trigger_delete_func(); */ CREATE OR REPLACE FUNCTION backfill_partitions() RETURNS VOID AS $$ DECLARE BEGIN IF EXISTS (SELECT 1 FROM items_default LIMIT 1) THEN RAISE NOTICE 'Creating new partitions and moving data from default'; CREATE TEMP TABLE items_default_tmp ON COMMIT DROP AS SELECT datetime, content FROM items_default; TRUNCATE items_default; PERFORM make_partitions(min(datetime), max(datetime)) FROM items_default_tmp; INSERT INTO items (content) SELECT content FROM items_default_tmp; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION items_trigger_stmt_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_stmt_trigger ON items; CREATE TRIGGER items_stmt_trigger AFTER INSERT OR UPDATE OR DELETE ON items FOR EACH STATEMENT EXECUTE PROCEDURE items_trigger_stmt_func(); /* View to get a table of available items partitions with date ranges */ --DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE OR REPLACE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; --DROP VIEW IF EXISTS items_partitions; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS items_partitions; CREATE VIEW items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base WHERE est_cnt >0 ORDER BY 2 desc; CREATE OR REPLACE FUNCTION items_by_partition( IN _where text DEFAULT 'TRUE', IN _dtrange tstzrange DEFAULT tstzrange('-infinity','infinity'), IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE partition_query text; main_query text; batchcount int; counter int := 0; p record; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange ASC ; $q$); ELSE partition_query := format($q$ SELECT 'items' as partition WHERE $1 IS NOT NULL; $q$); END IF; RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query USING (_dtrange) LOOP IF lower(_dtrange)::timestamptz > '-infinity' THEN _where := concat(_where,format(' AND datetime >= %L',lower(_dtrange)::timestamptz::text)); END IF; IF upper(_dtrange)::timestamptz < 'infinity' THEN _where := concat(_where,format(' AND datetime <= %L',upper(_dtrange)::timestamptz::text)); END IF; main_query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s LIMIT %s - $1 $q$, p.partition::text, _where, _orderby, _limit ); RAISE NOTICE 'Partition Query %', main_query; RAISE NOTICE '%', counter; RETURN QUERY EXECUTE main_query USING counter; GET DIAGNOSTICS batchcount = ROW_COUNT; counter := counter + batchcount; RAISE NOTICE 'FOUND %', batchcount; IF counter >= _limit THEN EXIT; END IF; RAISE NOTICE 'ADDED % FOR A TOTAL OF %', batchcount, counter; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION split_stac_path(IN path text, OUT col text, OUT dotpath text, OUT jspath text, OUT jspathtext text) AS $$ WITH col AS ( SELECT CASE WHEN split_part(path, '.', 1) IN ('id', 'stac_version', 'stac_extensions','geometry','properties','assets','collection_id','datetime','links', 'extra_fields') THEN split_part(path, '.', 1) ELSE 'properties' END AS col ), dp AS ( SELECT col, ltrim(replace(path, col , ''),'.') as dotpath FROM col ), paths AS ( SELECT col, dotpath, regexp_split_to_table(dotpath,E'\\.') as path FROM dp ) SELECT col, btrim(concat(col,'.',dotpath),'.'), CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, regexp_replace( CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, E'>([^>]*)$','>>\1' ) FROM paths group by col, dotpath; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions for searching items */ CREATE OR REPLACE FUNCTION sort_base( IN _sort jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]', OUT key text, OUT col text, OUT dir text, OUT rdir text, OUT sort text, OUT rsort text ) RETURNS SETOF RECORD AS $$ WITH sorts AS ( SELECT value->>'field' as key, (split_stac_path(value->>'field')).jspathtext as col, coalesce(upper(value->>'direction'),'ASC') as dir FROM jsonb_array_elements('[]'::jsonb || coalesce(_sort,'[{"field":"datetime","direction":"desc"}]') ) ) SELECT key, col, dir, CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END as rdir, concat(col, ' ', dir, ' NULLS LAST ') AS sort, concat(col,' ', CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END, ' NULLS LAST ') AS rsort FROM sorts UNION ALL SELECT 'id', 'id', 'DESC', 'ASC', 'id DESC', 'id ASC' ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(sort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION rsort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(rsort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS box3d AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION in_array_q(col text, arr jsonb) RETURNS text AS $$ SELECT CASE jsonb_typeof(arr) WHEN 'array' THEN format('%I = ANY(textarr(%L))', col, arr) ELSE format('%I = %L', col, arr) END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION count_by_delim(text, text) RETURNS int AS $$ SELECT count(*) FROM regexp_split_to_table($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$ DECLARE ret text := ''; op text; jp text; att_parts RECORD; val_str text; BEGIN val_str := lower(jsonb_build_object('a',val)->>'a'); RAISE NOTICE 'val_str %', val_str; att_parts := split_stac_path(att); op := CASE _op WHEN 'eq' THEN '=' WHEN 'ge' THEN '>=' WHEN 'gt' THEN '>' WHEN 'le' THEN '<=' WHEN 'lt' THEN '<' WHEN 'ne' THEN '!=' WHEN 'neq' THEN '!=' WHEN 'startsWith' THEN 'LIKE' WHEN 'endsWith' THEN 'LIKE' WHEN 'contains' THEN 'LIKE' ELSE _op END; val_str := CASE _op WHEN 'startsWith' THEN concat(val_str, '%') WHEN 'endsWith' THEN concat('%', val_str) WHEN 'contains' THEN concat('%',val_str,'%') ELSE val_str END; RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.'); IF op = '=' AND att_parts.col = 'properties' --AND count_by_delim(att_parts.dotpath,'\.') = 2 THEN -- use jsonpath query to leverage index for eqaulity tests on single level deep properties jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb)); raise notice 'jp: %', jp; ret := format($q$ properties @? %L $q$, jp); ELSIF jsonb_typeof(val) = 'number' THEN ret := format('(%s)::numeric %s %s', att_parts.jspathtext, op, val); ELSE ret := format('%s %s %L', att_parts.jspathtext, op, val_str); END IF; RAISE NOTICE 'Op Query: %', ret; return ret; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION stac_query(_query jsonb) RETURNS TEXT[] AS $$ DECLARE qa text[]; att text; ops jsonb; op text; val jsonb; BEGIN FOR att, ops IN SELECT key, value FROM jsonb_each(_query) LOOP FOR op, val IN SELECT key, value FROM jsonb_each(ops) LOOP qa := array_append(qa, stac_query_op(att,op, val)); RAISE NOTICE '% % %', att, op, val; END LOOP; END LOOP; RETURN qa; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION filter_by_order(item_id text, _sort jsonb, _type text) RETURNS text AS $$ DECLARE item item; BEGIN SELECT * INTO item FROM items WHERE id=item_id; RETURN filter_by_order(item, _sort, _type); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; -- Used to create filters used for paging using the items id from the token CREATE OR REPLACE FUNCTION filter_by_order(_item item, _sort jsonb, _type text) RETURNS text AS $$ DECLARE sorts RECORD; filts text[]; itemval text; op text; idop text; ret text; eq_flag text; _item_j jsonb := to_jsonb(_item); BEGIN FOR sorts IN SELECT * FROM sort_base(_sort) LOOP IF sorts.col = 'datetime' THEN CONTINUE; END IF; IF sorts.col='id' AND _type IN ('prev','next') THEN eq_flag := ''; ELSE eq_flag := '='; END IF; op := concat( CASE WHEN _type in ('prev','first') AND sorts.dir = 'ASC' THEN '<' WHEN _type in ('last','next') AND sorts.dir = 'ASC' THEN '>' WHEN _type in ('prev','first') AND sorts.dir = 'DESC' THEN '>' WHEN _type in ('last','next') AND sorts.dir = 'DESC' THEN '<' END, eq_flag ); IF _item_j ? sorts.col THEN filts = array_append(filts, format('%s %s %L', sorts.col, op, _item_j->>sorts.col)); END IF; END LOOP; ret := coalesce(array_to_string(filts,' AND '), 'TRUE'); RAISE NOTICE 'Order Filter %', ret; RETURN ret; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION search_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ WITH t AS ( SELECT i, row_number() over () as r FROM jsonb_array_elements(j) i ), o AS ( SELECT i FROM t ORDER BY r DESC ) SELECT jsonb_agg(i) from o ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS SETOF jsonb AS $$ DECLARE qstart timestamptz := clock_timestamp(); _sort text := ''; _rsort text := ''; _limit int := 10; _geom geometry; qa text[]; pq text[]; query text; pq_prop record; pq_op record; prev_id text := NULL; next_id text := NULL; whereq text := 'TRUE'; links jsonb := '[]'::jsonb; token text; tok_val text; tok_q text := 'TRUE'; tok_sort text; first_id text; first_dt timestamptz; last_id text; sort text; rsort text; dt text[]; dqa text[]; dq text; mq_where text; startdt timestamptz; enddt timestamptz; item items%ROWTYPE; counter int := 0; batchcount int; month timestamptz; m record; _dtrange tstzrange := tstzrange('-infinity','infinity'); _dtsort text; _token_dtrange tstzrange := tstzrange('-infinity','infinity'); _token_record items%ROWTYPE; is_prev boolean := false; includes text[]; excludes text[]; BEGIN -- Create table from sort query of items to sort CREATE TEMP TABLE pgstac_tmp_sorts ON COMMIT DROP AS SELECT * FROM sort_base(_search->'sortby'); -- Get the datetime sort direction, necessary for efficient cycling through partitions SELECT INTO _dtsort dir FROM pgstac_tmp_sorts WHERE key='datetime'; RAISE NOTICE '_dtsort: %',_dtsort; SELECT INTO _sort string_agg(s.sort,', ') FROM pgstac_tmp_sorts s; SELECT INTO _rsort string_agg(s.rsort,', ') FROM pgstac_tmp_sorts s; tok_sort := _sort; -- Get datetime from query as a tstzrange IF _search ? 'datetime' THEN _dtrange := search_dtrange(_search->'datetime'); _token_dtrange := _dtrange; END IF; -- Get the paging token IF _search ? 'token' THEN token := _search->>'token'; tok_val := substr(token,6); IF starts_with(token, 'prev:') THEN is_prev := true; END IF; SELECT INTO _token_record * FROM items WHERE id=tok_val; IF (is_prev AND _dtsort = 'DESC') OR (not is_prev AND _dtsort = 'ASC') THEN _token_dtrange := tstzrange(_token_record.datetime, 'infinity'); ELSIF _dtsort IS NOT NULL THEN _token_dtrange := tstzrange('-infinity',_token_record.datetime); END IF; IF is_prev THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'first'); _sort := _rsort; ELSIF starts_with(token, 'next:') THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'last'); END IF; END IF; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'tok_q: % _token_dtrange: %', tok_q, _token_dtrange; IF _search ? 'ids' THEN RAISE NOTICE 'searching solely based on ids... %',_search; qa := array_append(qa, in_array_q('id', _search->'ids')); ELSE IF _search ? 'intersects' THEN _geom := ST_SetSRID(ST_GeomFromGeoJSON(_search->>'intersects'), 4326); ELSIF _search ? 'bbox' THEN _geom := bbox_geom(_search->'bbox'); END IF; IF _geom IS NOT NULL THEN qa := array_append(qa, format('st_intersects(geometry, %L::geometry)',_geom)); END IF; IF _search ? 'collections' THEN qa := array_append(qa, in_array_q('collection_id', _search->'collections')); END IF; IF _search ? 'query' THEN qa := array_cat(qa, stac_query(_search->'query') ); END IF; END IF; IF _search ? 'limit' THEN _limit := (_search->>'limit')::int; END IF; IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; RAISE NOTICE 'Includes: %, Excludes: %', includes, excludes; END IF; whereq := COALESCE(array_to_string(qa,' AND '),' TRUE '); dq := COALESCE(array_to_string(dqa,' AND '),' TRUE '); RAISE NOTICE 'timing before temp table: %', age(clock_timestamp(), qstart); CREATE TEMP TABLE results_page ON COMMIT DROP AS SELECT * FROM items_by_partition( concat(whereq, ' AND ', tok_q), _token_dtrange, _sort, _limit + 1 ); RAISE NOTICE 'timing after temp table: %', age(clock_timestamp(), qstart); RAISE NOTICE 'timing before min/max: %', age(clock_timestamp(), qstart); IF is_prev THEN SELECT INTO last_id, first_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; ELSE SELECT INTO first_id, last_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; END IF; RAISE NOTICE 'firstid: %, lastid %', first_id, last_id; RAISE NOTICE 'timing after min/max: %', age(clock_timestamp(), qstart); IF counter > _limit THEN next_id := last_id; RAISE NOTICE 'next_id: %', next_id; ELSE RAISE NOTICE 'No more next'; END IF; IF tok_q = 'TRUE' THEN RAISE NOTICE 'Not a paging query, no previous item'; ELSE RAISE NOTICE 'Getting previous item id'; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); SELECT INTO _token_record * FROM items WHERE id=first_id; IF _dtsort = 'DESC' THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSE _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; RAISE NOTICE '% %', _token_dtrange, _dtrange; SELECT id INTO prev_id FROM items_by_partition( concat(whereq, ' AND ', filter_by_order(first_id, _search->'sortby', 'prev')), _token_dtrange, _rsort, 1 ); RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'prev_id: %', prev_id; END IF; RETURN QUERY WITH features AS ( SELECT filter_jsonb(content, includes, excludes) as content FROM results_page LIMIT _limit ), j AS (SELECT jsonb_agg(content) as feature_arr FROM features) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce ( CASE WHEN is_prev THEN flip_jsonb_array(feature_arr) ELSE feature_arr END ,'[]'::jsonb), 'links', links, 'timeStamp', now(), 'next', next_id, 'prev', prev_id ) FROM j ; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; INSERT INTO migrations (version) VALUES ('0.2.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.2.5-0.2.7.sql ================================================ SET SEARCH_PATH TO pgstac, public; BEGIN; CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$ DECLARE ret text := ''; op text; jp text; att_parts RECORD; val_str text; prop_path text; BEGIN val_str := lower(jsonb_build_object('a',val)->>'a'); RAISE NOTICE 'val_str %', val_str; att_parts := split_stac_path(att); prop_path := replace(att_parts.dotpath, 'properties.', ''); op := CASE _op WHEN 'eq' THEN '=' WHEN 'gte' THEN '>=' WHEN 'gt' THEN '>' WHEN 'lte' THEN '<=' WHEN 'lt' THEN '<' WHEN 'ne' THEN '!=' WHEN 'neq' THEN '!=' WHEN 'startsWith' THEN 'LIKE' WHEN 'endsWith' THEN 'LIKE' WHEN 'contains' THEN 'LIKE' ELSE _op END; val_str := CASE _op WHEN 'startsWith' THEN concat(val_str, '%') WHEN 'endsWith' THEN concat('%', val_str) WHEN 'contains' THEN concat('%',val_str,'%') ELSE val_str END; RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.'); IF op = '=' AND att_parts.col = 'properties' --AND count_by_delim(att_parts.dotpath,'\.') = 2 THEN -- use jsonpath query to leverage index for eqaulity tests on single level deep properties jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb)); raise notice 'jp: %', jp; ret := format($q$ properties @? %L $q$, jp); ELSIF jsonb_typeof(val) = 'number' THEN ret := format('properties ? %L AND (%s)::numeric %s %s', prop_path, att_parts.jspathtext, op, val); ELSE ret := format('properties ? %L AND %s %s %L', prop_path ,att_parts.jspathtext, op, val_str); END IF; RAISE NOTICE 'Op Query: %', ret; return ret; END; $$ LANGUAGE PLPGSQL; INSERT INTO migrations (version) VALUES ('0.2.7'); COMMIT; ================================================ FILE: src/pgstac/migrations/pgstac.0.2.5.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS partman; CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT ARRAY(SELECT jsonb_array_elements_text(_js)); $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* converts a jsonb text array to comma delimited list of identifer quoted useful for constructing column lists for selects */ CREATE OR REPLACE FUNCTION array_idents(_js jsonb) RETURNS text AS $$ SELECT string_agg(quote_ident(v),',') FROM jsonb_array_elements_text(_js) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT (value->'properties'->>'datetime')::timestamptz; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION properties_idx(_in jsonb) RETURNS jsonb AS $$ WITH t AS ( select array_to_string(path,'.') as path, lower(value::text)::jsonb as lowerval FROM jsonb_val_paths(_in) WHERE array_to_string(path,'.') not in ('datetime') ) SELECT jsonb_object_agg(path, lowerval) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; /* CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ BEGIN IF pg_trigger_depth() = 1 THEN PERFORM create_collection(NEW.content); RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger BEFORE INSERT ON collections FOR EACH ROW EXECUTE PROCEDURE collections_trigger_func(); */ SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS items ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED NOT NULL, geometry geometry GENERATED ALWAYS AS (stac_geom(content)) STORED NOT NULL, properties jsonb GENERATED ALWAYS as (properties_idx(content->'properties')) STORED, collection_id text GENERATED ALWAYS AS (content->>'collection') STORED NOT NULL, datetime timestamptz GENERATED ALWAYS AS (stac_datetime(content)) STORED NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (stac_datetime(content)) ; ALTER TABLE items ADD constraint items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) DEFERRABLE; CREATE TABLE items_template ( LIKE items ); ALTER TABLE items_template ADD PRIMARY KEY (id); /* CREATE TABLE IF NOT EXISTS items_search ( id text NOT NULL, geometry geometry NOT NULL, properties jsonb, collection_id text NOT NULL, datetime timestamptz NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE TABLE IF NOT EXISTS items_search_template ( LIKE items_search ) ; ALTER TABLE items_search_template ADD PRIMARY KEY (id); */ DELETE from partman.part_config WHERE parent_table = 'pgstac.items'; SELECT partman.create_parent( 'pgstac.items', 'datetime', 'native', 'weekly', p_template_table := 'pgstac.items_template', p_premake := 4 ); CREATE OR REPLACE FUNCTION make_partitions(st timestamptz, et timestamptz DEFAULT NULL) RETURNS BOOL AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', coalesce(et, st)), '1 week'::interval ) w ), w AS (SELECT array_agg(w) as w FROM t) SELECT CASE WHEN w IS NULL THEN NULL ELSE partman.create_partition_time('pgstac.items', w, true) END FROM w; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_partition(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE INDEX "datetime_id_idx" ON items (datetime, id); CREATE INDEX "properties_idx" ON items USING GIN (properties); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE TYPE item AS ( id text, geometry geometry, properties JSONB, collection_id text, datetime timestamptz ); /* Converts single feature into an items row */ /* CREATE OR REPLACE FUNCTION feature_to_item(value jsonb) RETURNS item AS $$ SELECT value->>'id' as id, CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry, properties_idx(value ->'properties') as properties, value->>'collection' as collection_id, (value->'properties'->>'datetime')::timestamptz as datetime ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; */ /* Takes a single feature, an array of features, or a feature collection and returns a set up individual items rows */ /* CREATE OR REPLACE FUNCTION features_to_items(value jsonb) RETURNS SETOF item AS $$ WITH features AS ( SELECT jsonb_array_elements( CASE WHEN jsonb_typeof(value) = 'array' THEN value WHEN value->>'type' = 'Feature' THEN '[]'::jsonb || value WHEN value->>'type' = 'FeatureCollection' THEN value->'features' ELSE NULL END ) as value ) SELECT feature_to_item(value) FROM features ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; */ CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ SELECT make_partitions(stac_datetime(data)); INSERT INTO items (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ DECLARE partition text; q text; newcontent jsonb; BEGIN PERFORM make_partitions(stac_datetime(data)); partition := get_partition(stac_datetime(data)); q := format($q$ INSERT INTO %I (content) VALUES ($1) ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content WHERE %I.content IS DISTINCT FROM EXCLUDED.content RETURNING content; $q$, partition, partition); EXECUTE q INTO newcontent USING (data); RAISE NOTICE 'newcontent: %', newcontent; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP RAISE NOTICE 'Analyzing %', p; EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; /* Trigger Function to cascade inserts/updates/deletes from items table to items_search table */ /* ALTER TABLE items_search ADD CONSTRAINT items_search_fk FOREIGN KEY (id) REFERENCES items(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION items_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN IF TG_OP = 'UPDATE' THEN RAISE NOTICE 'DELETING % BEFORE UPDATE', OLD; DELETE FROM items_search WHERE id = OLD.id AND datetime = (OLD.content->'properties'->>'datetime')::timestamptz; END IF; INSERT INTO items_search SELECT * FROM feature_to_item(NEW.content); RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_insert_trigger ON items; CREATE TRIGGER items_insert_trigger AFTER INSERT ON items FOR EACH ROW EXECUTE PROCEDURE items_trigger_func(); DROP TRIGGER IF EXISTS items_update_trigger ON items; CREATE TRIGGER items_update_trigger AFTER UPDATE ON items FOR EACH ROW WHEN (NEW.content IS DISTINCT FROM OLD.content) EXECUTE PROCEDURE items_trigger_func(); */ /* Trigger Function to cascade inserts/updates/deletes from items table to items_search table */ /* CREATE OR REPLACE FUNCTION items_search_trigger_delete_func() RETURNS TRIGGER AS $$ DECLARE BEGIN RAISE NOTICE 'Deleting from items_search: % Depth: %', OLD, pg_trigger_depth(); IF pg_trigger_depth()<3 THEN RAISE NOTICE 'DELETING WITH datetime'; DELETE FROM items_search WHERE id=OLD.id AND datetime=OLD.datetime; RETURN NULL; END IF; RETURN OLD; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_search_delete_trigger ON items_search; CREATE TRIGGER items_search_delete_trigger BEFORE DELETE ON items_search FOR EACH ROW EXECUTE PROCEDURE items_search_trigger_delete_func(); */ CREATE OR REPLACE FUNCTION backfill_partitions() RETURNS VOID AS $$ DECLARE BEGIN IF EXISTS (SELECT 1 FROM items_default LIMIT 1) THEN RAISE NOTICE 'Creating new partitions and moving data from default'; CREATE TEMP TABLE items_default_tmp ON COMMIT DROP AS SELECT datetime, content FROM items_default; TRUNCATE items_default; PERFORM make_partitions(min(datetime), max(datetime)) FROM items_default_tmp; INSERT INTO items (content) SELECT content FROM items_default_tmp; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION items_trigger_stmt_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_stmt_trigger ON items; CREATE TRIGGER items_stmt_trigger AFTER INSERT OR UPDATE OR DELETE ON items FOR EACH STATEMENT EXECUTE PROCEDURE items_trigger_stmt_func(); /* View to get a table of available items partitions with date ranges */ --DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE OR REPLACE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; --DROP VIEW IF EXISTS items_partitions; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS items_partitions; CREATE VIEW items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base WHERE est_cnt >0 ORDER BY 2 desc; CREATE OR REPLACE FUNCTION items_by_partition( IN _where text DEFAULT 'TRUE', IN _dtrange tstzrange DEFAULT tstzrange('-infinity','infinity'), IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE partition_query text; main_query text; batchcount int; counter int := 0; p record; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange ASC ; $q$); ELSE partition_query := format($q$ SELECT 'items' as partition WHERE $1 IS NOT NULL; $q$); END IF; RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query USING (_dtrange) LOOP IF lower(_dtrange)::timestamptz > '-infinity' THEN _where := concat(_where,format(' AND datetime >= %L',lower(_dtrange)::timestamptz::text)); END IF; IF upper(_dtrange)::timestamptz < 'infinity' THEN _where := concat(_where,format(' AND datetime <= %L',upper(_dtrange)::timestamptz::text)); END IF; main_query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s LIMIT %s - $1 $q$, p.partition::text, _where, _orderby, _limit ); RAISE NOTICE 'Partition Query %', main_query; RAISE NOTICE '%', counter; RETURN QUERY EXECUTE main_query USING counter; GET DIAGNOSTICS batchcount = ROW_COUNT; counter := counter + batchcount; RAISE NOTICE 'FOUND %', batchcount; IF counter >= _limit THEN EXIT; END IF; RAISE NOTICE 'ADDED % FOR A TOTAL OF %', batchcount, counter; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION split_stac_path(IN path text, OUT col text, OUT dotpath text, OUT jspath text, OUT jspathtext text) AS $$ WITH col AS ( SELECT CASE WHEN split_part(path, '.', 1) IN ('id', 'stac_version', 'stac_extensions','geometry','properties','assets','collection_id','datetime','links', 'extra_fields') THEN split_part(path, '.', 1) ELSE 'properties' END AS col ), dp AS ( SELECT col, ltrim(replace(path, col , ''),'.') as dotpath FROM col ), paths AS ( SELECT col, dotpath, regexp_split_to_table(dotpath,E'\\.') as path FROM dp ) SELECT col, btrim(concat(col,'.',dotpath),'.'), CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, regexp_replace( CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, E'>([^>]*)$','>>\1' ) FROM paths group by col, dotpath; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions for searching items */ CREATE OR REPLACE FUNCTION sort_base( IN _sort jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]', OUT key text, OUT col text, OUT dir text, OUT rdir text, OUT sort text, OUT rsort text ) RETURNS SETOF RECORD AS $$ WITH sorts AS ( SELECT value->>'field' as key, (split_stac_path(value->>'field')).jspathtext as col, coalesce(upper(value->>'direction'),'ASC') as dir FROM jsonb_array_elements('[]'::jsonb || coalesce(_sort,'[{"field":"datetime","direction":"desc"}]') ) ) SELECT key, col, dir, CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END as rdir, concat(col, ' ', dir, ' NULLS LAST ') AS sort, concat(col,' ', CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END, ' NULLS LAST ') AS rsort FROM sorts UNION ALL SELECT 'id', 'id', 'DESC', 'ASC', 'id DESC', 'id ASC' ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(sort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION rsort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(rsort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION in_array_q(col text, arr jsonb) RETURNS text AS $$ SELECT CASE jsonb_typeof(arr) WHEN 'array' THEN format('%I = ANY(textarr(%L))', col, arr) ELSE format('%I = %L', col, arr) END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION count_by_delim(text, text) RETURNS int AS $$ SELECT count(*) FROM regexp_split_to_table($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$ DECLARE ret text := ''; op text; jp text; att_parts RECORD; val_str text; prop_path text; BEGIN val_str := lower(jsonb_build_object('a',val)->>'a'); RAISE NOTICE 'val_str %', val_str; att_parts := split_stac_path(att); prop_path := replace(att_parts.dotpath, 'properties.', ''); op := CASE _op WHEN 'eq' THEN '=' WHEN 'gte' THEN '>=' WHEN 'gt' THEN '>' WHEN 'lte' THEN '<=' WHEN 'lt' THEN '<' WHEN 'ne' THEN '!=' WHEN 'neq' THEN '!=' WHEN 'startsWith' THEN 'LIKE' WHEN 'endsWith' THEN 'LIKE' WHEN 'contains' THEN 'LIKE' ELSE _op END; val_str := CASE _op WHEN 'startsWith' THEN concat(val_str, '%') WHEN 'endsWith' THEN concat('%', val_str) WHEN 'contains' THEN concat('%',val_str,'%') ELSE val_str END; RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.'); IF op = '=' AND att_parts.col = 'properties' --AND count_by_delim(att_parts.dotpath,'\.') = 2 THEN -- use jsonpath query to leverage index for eqaulity tests on single level deep properties jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb)); raise notice 'jp: %', jp; ret := format($q$ properties @? %L $q$, jp); ELSIF jsonb_typeof(val) = 'number' THEN ret := format('properties ? %L AND (%s)::numeric %s %s', prop_path, att_parts.jspathtext, op, val); ELSE ret := format('properties ? %L AND %s %s %L', prop_path ,att_parts.jspathtext, op, val_str); END IF; RAISE NOTICE 'Op Query: %', ret; return ret; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION stac_query(_query jsonb) RETURNS TEXT[] AS $$ DECLARE qa text[]; att text; ops jsonb; op text; val jsonb; BEGIN FOR att, ops IN SELECT key, value FROM jsonb_each(_query) LOOP FOR op, val IN SELECT key, value FROM jsonb_each(ops) LOOP qa := array_append(qa, stac_query_op(att,op, val)); RAISE NOTICE '% % %', att, op, val; END LOOP; END LOOP; RETURN qa; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION filter_by_order(item_id text, _sort jsonb, _type text) RETURNS text AS $$ DECLARE item item; BEGIN SELECT * INTO item FROM items WHERE id=item_id; RETURN filter_by_order(item, _sort, _type); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; -- Used to create filters used for paging using the items id from the token CREATE OR REPLACE FUNCTION filter_by_order(_item item, _sort jsonb, _type text) RETURNS text AS $$ DECLARE sorts RECORD; filts text[]; itemval text; op text; idop text; ret text; eq_flag text; _item_j jsonb := to_jsonb(_item); BEGIN FOR sorts IN SELECT * FROM sort_base(_sort) LOOP IF sorts.col = 'datetime' THEN CONTINUE; END IF; IF sorts.col='id' AND _type IN ('prev','next') THEN eq_flag := ''; ELSE eq_flag := '='; END IF; op := concat( CASE WHEN _type in ('prev','first') AND sorts.dir = 'ASC' THEN '<' WHEN _type in ('last','next') AND sorts.dir = 'ASC' THEN '>' WHEN _type in ('prev','first') AND sorts.dir = 'DESC' THEN '>' WHEN _type in ('last','next') AND sorts.dir = 'DESC' THEN '<' END, eq_flag ); IF _item_j ? sorts.col THEN filts = array_append(filts, format('%s %s %L', sorts.col, op, _item_j->>sorts.col)); END IF; END LOOP; ret := coalesce(array_to_string(filts,' AND '), 'TRUE'); RAISE NOTICE 'Order Filter %', ret; RETURN ret; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION search_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ WITH t AS ( SELECT i, row_number() over () as r FROM jsonb_array_elements(j) i ), o AS ( SELECT i FROM t ORDER BY r DESC ) SELECT jsonb_agg(i) from o ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS SETOF jsonb AS $$ DECLARE qstart timestamptz := clock_timestamp(); _sort text := ''; _rsort text := ''; _limit int := 10; _geom geometry; qa text[]; pq text[]; query text; pq_prop record; pq_op record; prev_id text := NULL; next_id text := NULL; whereq text := 'TRUE'; links jsonb := '[]'::jsonb; token text; tok_val text; tok_q text := 'TRUE'; tok_sort text; first_id text; first_dt timestamptz; last_id text; sort text; rsort text; dt text[]; dqa text[]; dq text; mq_where text; startdt timestamptz; enddt timestamptz; item items%ROWTYPE; counter int := 0; batchcount int; month timestamptz; m record; _dtrange tstzrange := tstzrange('-infinity','infinity'); _dtsort text; _token_dtrange tstzrange := tstzrange('-infinity','infinity'); _token_record items%ROWTYPE; is_prev boolean := false; includes text[]; excludes text[]; BEGIN -- Create table from sort query of items to sort CREATE TEMP TABLE pgstac_tmp_sorts ON COMMIT DROP AS SELECT * FROM sort_base(_search->'sortby'); -- Get the datetime sort direction, necessary for efficient cycling through partitions SELECT INTO _dtsort dir FROM pgstac_tmp_sorts WHERE key='datetime'; RAISE NOTICE '_dtsort: %',_dtsort; SELECT INTO _sort string_agg(s.sort,', ') FROM pgstac_tmp_sorts s; SELECT INTO _rsort string_agg(s.rsort,', ') FROM pgstac_tmp_sorts s; tok_sort := _sort; -- Get datetime from query as a tstzrange IF _search ? 'datetime' THEN _dtrange := search_dtrange(_search->'datetime'); _token_dtrange := _dtrange; END IF; -- Get the paging token IF _search ? 'token' THEN token := _search->>'token'; tok_val := substr(token,6); IF starts_with(token, 'prev:') THEN is_prev := true; END IF; SELECT INTO _token_record * FROM items WHERE id=tok_val; IF (is_prev AND _dtsort = 'DESC') OR (not is_prev AND _dtsort = 'ASC') THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSIF _dtsort IS NOT NULL THEN _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; IF is_prev THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'first'); _sort := _rsort; ELSIF starts_with(token, 'next:') THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'last'); END IF; END IF; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'tok_q: % _token_dtrange: %', tok_q, _token_dtrange; IF _search ? 'ids' THEN RAISE NOTICE 'searching solely based on ids... %',_search; qa := array_append(qa, in_array_q('id', _search->'ids')); ELSE IF _search ? 'intersects' THEN _geom := ST_SetSRID(ST_GeomFromGeoJSON(_search->>'intersects'), 4326); ELSIF _search ? 'bbox' THEN _geom := bbox_geom(_search->'bbox'); END IF; IF _geom IS NOT NULL THEN qa := array_append(qa, format('st_intersects(geometry, %L::geometry)',_geom)); END IF; IF _search ? 'collections' THEN qa := array_append(qa, in_array_q('collection_id', _search->'collections')); END IF; IF _search ? 'query' THEN qa := array_cat(qa, stac_query(_search->'query') ); END IF; END IF; IF _search ? 'limit' THEN _limit := (_search->>'limit')::int; END IF; IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; RAISE NOTICE 'Includes: %, Excludes: %', includes, excludes; END IF; whereq := COALESCE(array_to_string(qa,' AND '),' TRUE '); dq := COALESCE(array_to_string(dqa,' AND '),' TRUE '); RAISE NOTICE 'timing before temp table: %', age(clock_timestamp(), qstart); CREATE TEMP TABLE results_page ON COMMIT DROP AS SELECT * FROM items_by_partition( concat(whereq, ' AND ', tok_q), _token_dtrange, _sort, _limit + 1 ); RAISE NOTICE 'timing after temp table: %', age(clock_timestamp(), qstart); RAISE NOTICE 'timing before min/max: %', age(clock_timestamp(), qstart); IF is_prev THEN SELECT INTO last_id, first_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; ELSE SELECT INTO first_id, last_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; END IF; RAISE NOTICE 'firstid: %, lastid %', first_id, last_id; RAISE NOTICE 'timing after min/max: %', age(clock_timestamp(), qstart); IF counter > _limit THEN next_id := last_id; RAISE NOTICE 'next_id: %', next_id; ELSE RAISE NOTICE 'No more next'; END IF; IF tok_q = 'TRUE' THEN RAISE NOTICE 'Not a paging query, no previous item'; ELSE RAISE NOTICE 'Getting previous item id'; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); SELECT INTO _token_record * FROM items WHERE id=first_id; IF _dtsort = 'DESC' THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSE _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; RAISE NOTICE '% %', _token_dtrange, _dtrange; SELECT id INTO prev_id FROM items_by_partition( concat(whereq, ' AND ', filter_by_order(first_id, _search->'sortby', 'prev')), _token_dtrange, _rsort, 1 ); RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'prev_id: %', prev_id; END IF; RETURN QUERY WITH features AS ( SELECT filter_jsonb(content, includes, excludes) as content FROM results_page LIMIT _limit ), j AS (SELECT jsonb_agg(content) as feature_arr FROM features) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce ( CASE WHEN is_prev THEN flip_jsonb_array(feature_arr) ELSE feature_arr END ,'[]'::jsonb), 'links', links, 'timeStamp', now(), 'next', next_id, 'prev', prev_id ) FROM j ; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; INSERT INTO migrations (version) VALUES ('0.2.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.2.7-0.2.8.sql ================================================ SET SEARCH_PATH TO pgstac, public; BEGIN; INSERT INTO migrations (version) VALUES ('0.2.8'); COMMIT; ================================================ FILE: src/pgstac/migrations/pgstac.0.2.7.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS partman; CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT ARRAY(SELECT jsonb_array_elements_text(_js)); $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* converts a jsonb text array to comma delimited list of identifer quoted useful for constructing column lists for selects */ CREATE OR REPLACE FUNCTION array_idents(_js jsonb) RETURNS text AS $$ SELECT string_agg(quote_ident(v),',') FROM jsonb_array_elements_text(_js) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT (value->'properties'->>'datetime')::timestamptz; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION properties_idx(_in jsonb) RETURNS jsonb AS $$ WITH t AS ( select array_to_string(path,'.') as path, lower(value::text)::jsonb as lowerval FROM jsonb_val_paths(_in) WHERE array_to_string(path,'.') not in ('datetime') ) SELECT jsonb_object_agg(path, lowerval) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; /* CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ BEGIN IF pg_trigger_depth() = 1 THEN PERFORM create_collection(NEW.content); RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger BEFORE INSERT ON collections FOR EACH ROW EXECUTE PROCEDURE collections_trigger_func(); */ SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS items ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED NOT NULL, geometry geometry GENERATED ALWAYS AS (stac_geom(content)) STORED NOT NULL, properties jsonb GENERATED ALWAYS as (properties_idx(content->'properties')) STORED, collection_id text GENERATED ALWAYS AS (content->>'collection') STORED NOT NULL, datetime timestamptz GENERATED ALWAYS AS (stac_datetime(content)) STORED NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (stac_datetime(content)) ; ALTER TABLE items ADD constraint items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) DEFERRABLE; CREATE TABLE items_template ( LIKE items ); ALTER TABLE items_template ADD PRIMARY KEY (id); /* CREATE TABLE IF NOT EXISTS items_search ( id text NOT NULL, geometry geometry NOT NULL, properties jsonb, collection_id text NOT NULL, datetime timestamptz NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE TABLE IF NOT EXISTS items_search_template ( LIKE items_search ) ; ALTER TABLE items_search_template ADD PRIMARY KEY (id); */ DELETE from partman.part_config WHERE parent_table = 'pgstac.items'; SELECT partman.create_parent( 'pgstac.items', 'datetime', 'native', 'weekly', p_template_table := 'pgstac.items_template', p_premake := 4 ); CREATE OR REPLACE FUNCTION make_partitions(st timestamptz, et timestamptz DEFAULT NULL) RETURNS BOOL AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', coalesce(et, st)), '1 week'::interval ) w ), w AS (SELECT array_agg(w) as w FROM t) SELECT CASE WHEN w IS NULL THEN NULL ELSE partman.create_partition_time('pgstac.items', w, true) END FROM w; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_partition(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE INDEX "datetime_id_idx" ON items (datetime, id); CREATE INDEX "properties_idx" ON items USING GIN (properties); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE TYPE item AS ( id text, geometry geometry, properties JSONB, collection_id text, datetime timestamptz ); /* Converts single feature into an items row */ /* CREATE OR REPLACE FUNCTION feature_to_item(value jsonb) RETURNS item AS $$ SELECT value->>'id' as id, CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry, properties_idx(value ->'properties') as properties, value->>'collection' as collection_id, (value->'properties'->>'datetime')::timestamptz as datetime ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; */ /* Takes a single feature, an array of features, or a feature collection and returns a set up individual items rows */ /* CREATE OR REPLACE FUNCTION features_to_items(value jsonb) RETURNS SETOF item AS $$ WITH features AS ( SELECT jsonb_array_elements( CASE WHEN jsonb_typeof(value) = 'array' THEN value WHEN value->>'type' = 'Feature' THEN '[]'::jsonb || value WHEN value->>'type' = 'FeatureCollection' THEN value->'features' ELSE NULL END ) as value ) SELECT feature_to_item(value) FROM features ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; */ CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ SELECT make_partitions(stac_datetime(data)); INSERT INTO items (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ DECLARE partition text; q text; newcontent jsonb; BEGIN PERFORM make_partitions(stac_datetime(data)); partition := get_partition(stac_datetime(data)); q := format($q$ INSERT INTO %I (content) VALUES ($1) ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content WHERE %I.content IS DISTINCT FROM EXCLUDED.content RETURNING content; $q$, partition, partition); EXECUTE q INTO newcontent USING (data); RAISE NOTICE 'newcontent: %', newcontent; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP RAISE NOTICE 'Analyzing %', p; EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; /* Trigger Function to cascade inserts/updates/deletes from items table to items_search table */ /* ALTER TABLE items_search ADD CONSTRAINT items_search_fk FOREIGN KEY (id) REFERENCES items(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION items_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN IF TG_OP = 'UPDATE' THEN RAISE NOTICE 'DELETING % BEFORE UPDATE', OLD; DELETE FROM items_search WHERE id = OLD.id AND datetime = (OLD.content->'properties'->>'datetime')::timestamptz; END IF; INSERT INTO items_search SELECT * FROM feature_to_item(NEW.content); RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_insert_trigger ON items; CREATE TRIGGER items_insert_trigger AFTER INSERT ON items FOR EACH ROW EXECUTE PROCEDURE items_trigger_func(); DROP TRIGGER IF EXISTS items_update_trigger ON items; CREATE TRIGGER items_update_trigger AFTER UPDATE ON items FOR EACH ROW WHEN (NEW.content IS DISTINCT FROM OLD.content) EXECUTE PROCEDURE items_trigger_func(); */ /* Trigger Function to cascade inserts/updates/deletes from items table to items_search table */ /* CREATE OR REPLACE FUNCTION items_search_trigger_delete_func() RETURNS TRIGGER AS $$ DECLARE BEGIN RAISE NOTICE 'Deleting from items_search: % Depth: %', OLD, pg_trigger_depth(); IF pg_trigger_depth()<3 THEN RAISE NOTICE 'DELETING WITH datetime'; DELETE FROM items_search WHERE id=OLD.id AND datetime=OLD.datetime; RETURN NULL; END IF; RETURN OLD; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_search_delete_trigger ON items_search; CREATE TRIGGER items_search_delete_trigger BEFORE DELETE ON items_search FOR EACH ROW EXECUTE PROCEDURE items_search_trigger_delete_func(); */ CREATE OR REPLACE FUNCTION backfill_partitions() RETURNS VOID AS $$ DECLARE BEGIN IF EXISTS (SELECT 1 FROM items_default LIMIT 1) THEN RAISE NOTICE 'Creating new partitions and moving data from default'; CREATE TEMP TABLE items_default_tmp ON COMMIT DROP AS SELECT datetime, content FROM items_default; TRUNCATE items_default; PERFORM make_partitions(min(datetime), max(datetime)) FROM items_default_tmp; INSERT INTO items (content) SELECT content FROM items_default_tmp; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION items_trigger_stmt_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_stmt_trigger ON items; CREATE TRIGGER items_stmt_trigger AFTER INSERT OR UPDATE OR DELETE ON items FOR EACH STATEMENT EXECUTE PROCEDURE items_trigger_stmt_func(); /* View to get a table of available items partitions with date ranges */ --DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE OR REPLACE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; --DROP VIEW IF EXISTS items_partitions; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS items_partitions; CREATE VIEW items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base WHERE est_cnt >0 ORDER BY 2 desc; CREATE OR REPLACE FUNCTION items_by_partition( IN _where text DEFAULT 'TRUE', IN _dtrange tstzrange DEFAULT tstzrange('-infinity','infinity'), IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE partition_query text; main_query text; batchcount int; counter int := 0; p record; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange ASC ; $q$); ELSE partition_query := format($q$ SELECT 'items' as partition WHERE $1 IS NOT NULL; $q$); END IF; RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query USING (_dtrange) LOOP IF lower(_dtrange)::timestamptz > '-infinity' THEN _where := concat(_where,format(' AND datetime >= %L',lower(_dtrange)::timestamptz::text)); END IF; IF upper(_dtrange)::timestamptz < 'infinity' THEN _where := concat(_where,format(' AND datetime <= %L',upper(_dtrange)::timestamptz::text)); END IF; main_query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s LIMIT %s - $1 $q$, p.partition::text, _where, _orderby, _limit ); RAISE NOTICE 'Partition Query %', main_query; RAISE NOTICE '%', counter; RETURN QUERY EXECUTE main_query USING counter; GET DIAGNOSTICS batchcount = ROW_COUNT; counter := counter + batchcount; RAISE NOTICE 'FOUND %', batchcount; IF counter >= _limit THEN EXIT; END IF; RAISE NOTICE 'ADDED % FOR A TOTAL OF %', batchcount, counter; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION split_stac_path(IN path text, OUT col text, OUT dotpath text, OUT jspath text, OUT jspathtext text) AS $$ WITH col AS ( SELECT CASE WHEN split_part(path, '.', 1) IN ('id', 'stac_version', 'stac_extensions','geometry','properties','assets','collection_id','datetime','links', 'extra_fields') THEN split_part(path, '.', 1) ELSE 'properties' END AS col ), dp AS ( SELECT col, ltrim(replace(path, col , ''),'.') as dotpath FROM col ), paths AS ( SELECT col, dotpath, regexp_split_to_table(dotpath,E'\\.') as path FROM dp ) SELECT col, btrim(concat(col,'.',dotpath),'.'), CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, regexp_replace( CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, E'>([^>]*)$','>>\1' ) FROM paths group by col, dotpath; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions for searching items */ CREATE OR REPLACE FUNCTION sort_base( IN _sort jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]', OUT key text, OUT col text, OUT dir text, OUT rdir text, OUT sort text, OUT rsort text ) RETURNS SETOF RECORD AS $$ WITH sorts AS ( SELECT value->>'field' as key, (split_stac_path(value->>'field')).jspathtext as col, coalesce(upper(value->>'direction'),'ASC') as dir FROM jsonb_array_elements('[]'::jsonb || coalesce(_sort,'[{"field":"datetime","direction":"desc"}]') ) ) SELECT key, col, dir, CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END as rdir, concat(col, ' ', dir, ' NULLS LAST ') AS sort, concat(col,' ', CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END, ' NULLS LAST ') AS rsort FROM sorts UNION ALL SELECT 'id', 'id', 'DESC', 'ASC', 'id DESC', 'id ASC' ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(sort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION rsort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(rsort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION in_array_q(col text, arr jsonb) RETURNS text AS $$ SELECT CASE jsonb_typeof(arr) WHEN 'array' THEN format('%I = ANY(textarr(%L))', col, arr) ELSE format('%I = %L', col, arr) END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION count_by_delim(text, text) RETURNS int AS $$ SELECT count(*) FROM regexp_split_to_table($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$ DECLARE ret text := ''; op text; jp text; att_parts RECORD; val_str text; prop_path text; BEGIN val_str := lower(jsonb_build_object('a',val)->>'a'); RAISE NOTICE 'val_str %', val_str; att_parts := split_stac_path(att); prop_path := replace(att_parts.dotpath, 'properties.', ''); op := CASE _op WHEN 'eq' THEN '=' WHEN 'gte' THEN '>=' WHEN 'gt' THEN '>' WHEN 'lte' THEN '<=' WHEN 'lt' THEN '<' WHEN 'ne' THEN '!=' WHEN 'neq' THEN '!=' WHEN 'startsWith' THEN 'LIKE' WHEN 'endsWith' THEN 'LIKE' WHEN 'contains' THEN 'LIKE' ELSE _op END; val_str := CASE _op WHEN 'startsWith' THEN concat(val_str, '%') WHEN 'endsWith' THEN concat('%', val_str) WHEN 'contains' THEN concat('%',val_str,'%') ELSE val_str END; RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.'); IF op = '=' AND att_parts.col = 'properties' --AND count_by_delim(att_parts.dotpath,'\.') = 2 THEN -- use jsonpath query to leverage index for eqaulity tests on single level deep properties jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb)); raise notice 'jp: %', jp; ret := format($q$ properties @? %L $q$, jp); ELSIF jsonb_typeof(val) = 'number' THEN ret := format('properties ? %L AND (%s)::numeric %s %s', prop_path, att_parts.jspathtext, op, val); ELSE ret := format('properties ? %L AND %s %s %L', prop_path ,att_parts.jspathtext, op, val_str); END IF; RAISE NOTICE 'Op Query: %', ret; return ret; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION stac_query(_query jsonb) RETURNS TEXT[] AS $$ DECLARE qa text[]; att text; ops jsonb; op text; val jsonb; BEGIN FOR att, ops IN SELECT key, value FROM jsonb_each(_query) LOOP FOR op, val IN SELECT key, value FROM jsonb_each(ops) LOOP qa := array_append(qa, stac_query_op(att,op, val)); RAISE NOTICE '% % %', att, op, val; END LOOP; END LOOP; RETURN qa; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION filter_by_order(item_id text, _sort jsonb, _type text) RETURNS text AS $$ DECLARE item item; BEGIN SELECT * INTO item FROM items WHERE id=item_id; RETURN filter_by_order(item, _sort, _type); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; -- Used to create filters used for paging using the items id from the token CREATE OR REPLACE FUNCTION filter_by_order(_item item, _sort jsonb, _type text) RETURNS text AS $$ DECLARE sorts RECORD; filts text[]; itemval text; op text; idop text; ret text; eq_flag text; _item_j jsonb := to_jsonb(_item); BEGIN FOR sorts IN SELECT * FROM sort_base(_sort) LOOP IF sorts.col = 'datetime' THEN CONTINUE; END IF; IF sorts.col='id' AND _type IN ('prev','next') THEN eq_flag := ''; ELSE eq_flag := '='; END IF; op := concat( CASE WHEN _type in ('prev','first') AND sorts.dir = 'ASC' THEN '<' WHEN _type in ('last','next') AND sorts.dir = 'ASC' THEN '>' WHEN _type in ('prev','first') AND sorts.dir = 'DESC' THEN '>' WHEN _type in ('last','next') AND sorts.dir = 'DESC' THEN '<' END, eq_flag ); IF _item_j ? sorts.col THEN filts = array_append(filts, format('%s %s %L', sorts.col, op, _item_j->>sorts.col)); END IF; END LOOP; ret := coalesce(array_to_string(filts,' AND '), 'TRUE'); RAISE NOTICE 'Order Filter %', ret; RETURN ret; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION search_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ WITH t AS ( SELECT i, row_number() over () as r FROM jsonb_array_elements(j) i ), o AS ( SELECT i FROM t ORDER BY r DESC ) SELECT jsonb_agg(i) from o ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS SETOF jsonb AS $$ DECLARE qstart timestamptz := clock_timestamp(); _sort text := ''; _rsort text := ''; _limit int := 10; _geom geometry; qa text[]; pq text[]; query text; pq_prop record; pq_op record; prev_id text := NULL; next_id text := NULL; whereq text := 'TRUE'; links jsonb := '[]'::jsonb; token text; tok_val text; tok_q text := 'TRUE'; tok_sort text; first_id text; first_dt timestamptz; last_id text; sort text; rsort text; dt text[]; dqa text[]; dq text; mq_where text; startdt timestamptz; enddt timestamptz; item items%ROWTYPE; counter int := 0; batchcount int; month timestamptz; m record; _dtrange tstzrange := tstzrange('-infinity','infinity'); _dtsort text; _token_dtrange tstzrange := tstzrange('-infinity','infinity'); _token_record items%ROWTYPE; is_prev boolean := false; includes text[]; excludes text[]; BEGIN -- Create table from sort query of items to sort CREATE TEMP TABLE pgstac_tmp_sorts ON COMMIT DROP AS SELECT * FROM sort_base(_search->'sortby'); -- Get the datetime sort direction, necessary for efficient cycling through partitions SELECT INTO _dtsort dir FROM pgstac_tmp_sorts WHERE key='datetime'; RAISE NOTICE '_dtsort: %',_dtsort; SELECT INTO _sort string_agg(s.sort,', ') FROM pgstac_tmp_sorts s; SELECT INTO _rsort string_agg(s.rsort,', ') FROM pgstac_tmp_sorts s; tok_sort := _sort; -- Get datetime from query as a tstzrange IF _search ? 'datetime' THEN _dtrange := search_dtrange(_search->'datetime'); _token_dtrange := _dtrange; END IF; -- Get the paging token IF _search ? 'token' THEN token := _search->>'token'; tok_val := substr(token,6); IF starts_with(token, 'prev:') THEN is_prev := true; END IF; SELECT INTO _token_record * FROM items WHERE id=tok_val; IF (is_prev AND _dtsort = 'DESC') OR (not is_prev AND _dtsort = 'ASC') THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSIF _dtsort IS NOT NULL THEN _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; IF is_prev THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'first'); _sort := _rsort; ELSIF starts_with(token, 'next:') THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'last'); END IF; END IF; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'tok_q: % _token_dtrange: %', tok_q, _token_dtrange; IF _search ? 'ids' THEN RAISE NOTICE 'searching solely based on ids... %',_search; qa := array_append(qa, in_array_q('id', _search->'ids')); ELSE IF _search ? 'intersects' THEN _geom := ST_SetSRID(ST_GeomFromGeoJSON(_search->>'intersects'), 4326); ELSIF _search ? 'bbox' THEN _geom := bbox_geom(_search->'bbox'); END IF; IF _geom IS NOT NULL THEN qa := array_append(qa, format('st_intersects(geometry, %L::geometry)',_geom)); END IF; IF _search ? 'collections' THEN qa := array_append(qa, in_array_q('collection_id', _search->'collections')); END IF; IF _search ? 'query' THEN qa := array_cat(qa, stac_query(_search->'query') ); END IF; END IF; IF _search ? 'limit' THEN _limit := (_search->>'limit')::int; END IF; IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; RAISE NOTICE 'Includes: %, Excludes: %', includes, excludes; END IF; whereq := COALESCE(array_to_string(qa,' AND '),' TRUE '); dq := COALESCE(array_to_string(dqa,' AND '),' TRUE '); RAISE NOTICE 'timing before temp table: %', age(clock_timestamp(), qstart); CREATE TEMP TABLE results_page ON COMMIT DROP AS SELECT * FROM items_by_partition( concat(whereq, ' AND ', tok_q), _token_dtrange, _sort, _limit + 1 ); RAISE NOTICE 'timing after temp table: %', age(clock_timestamp(), qstart); RAISE NOTICE 'timing before min/max: %', age(clock_timestamp(), qstart); IF is_prev THEN SELECT INTO last_id, first_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; ELSE SELECT INTO first_id, last_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; END IF; RAISE NOTICE 'firstid: %, lastid %', first_id, last_id; RAISE NOTICE 'timing after min/max: %', age(clock_timestamp(), qstart); IF counter > _limit THEN next_id := last_id; RAISE NOTICE 'next_id: %', next_id; ELSE RAISE NOTICE 'No more next'; END IF; IF tok_q = 'TRUE' THEN RAISE NOTICE 'Not a paging query, no previous item'; ELSE RAISE NOTICE 'Getting previous item id'; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); SELECT INTO _token_record * FROM items WHERE id=first_id; IF _dtsort = 'DESC' THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSE _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; RAISE NOTICE '% %', _token_dtrange, _dtrange; SELECT id INTO prev_id FROM items_by_partition( concat(whereq, ' AND ', filter_by_order(first_id, _search->'sortby', 'prev')), _token_dtrange, _rsort, 1 ); RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'prev_id: %', prev_id; END IF; RETURN QUERY WITH features AS ( SELECT filter_jsonb(content, includes, excludes) as content FROM results_page LIMIT _limit ), j AS (SELECT jsonb_agg(content) as feature_arr FROM features) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce ( CASE WHEN is_prev THEN flip_jsonb_array(feature_arr) ELSE feature_arr END ,'[]'::jsonb), 'links', links, 'timeStamp', now(), 'next', next_id, 'prev', prev_id ) FROM j ; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; INSERT INTO migrations (version) VALUES ('0.2.7'); ================================================ FILE: src/pgstac/migrations/pgstac.0.2.8-0.2.9.sql ================================================ SET SEARCH_PATH to pgstac, public; INSERT INTO migrations (version) VALUES ('0.2.9'); ================================================ FILE: src/pgstac/migrations/pgstac.0.2.8.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS partman; CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT ARRAY(SELECT jsonb_array_elements_text(_js)); $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* converts a jsonb text array to comma delimited list of identifer quoted useful for constructing column lists for selects */ CREATE OR REPLACE FUNCTION array_idents(_js jsonb) RETURNS text AS $$ SELECT string_agg(quote_ident(v),',') FROM jsonb_array_elements_text(_js) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT (value->'properties'->>'datetime')::timestamptz; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION properties_idx(_in jsonb) RETURNS jsonb AS $$ WITH t AS ( select array_to_string(path,'.') as path, lower(value::text)::jsonb as lowerval FROM jsonb_val_paths(_in) WHERE array_to_string(path,'.') not in ('datetime') ) SELECT jsonb_object_agg(path, lowerval) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS items ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED NOT NULL, geometry geometry GENERATED ALWAYS AS (stac_geom(content)) STORED NOT NULL, properties jsonb GENERATED ALWAYS as (properties_idx(content->'properties')) STORED, collection_id text GENERATED ALWAYS AS (content->>'collection') STORED NOT NULL, datetime timestamptz GENERATED ALWAYS AS (stac_datetime(content)) STORED NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (stac_datetime(content)) ; ALTER TABLE items ADD constraint items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) DEFERRABLE; CREATE TABLE items_template ( LIKE items ); ALTER TABLE items_template ADD PRIMARY KEY (id); DELETE from partman.part_config WHERE parent_table = 'pgstac.items'; SELECT partman.create_parent( 'pgstac.items', 'datetime', 'native', 'weekly', p_template_table := 'pgstac.items_template', p_premake := 4 ); CREATE OR REPLACE FUNCTION make_partitions(st timestamptz, et timestamptz DEFAULT NULL) RETURNS BOOL AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', coalesce(et, st)), '1 week'::interval ) w ), w AS (SELECT array_agg(w) as w FROM t) SELECT CASE WHEN w IS NULL THEN NULL ELSE partman.create_partition_time('pgstac.items', w, true) END FROM w; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_partition(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE INDEX "datetime_id_idx" ON items (datetime, id); CREATE INDEX "properties_idx" ON items USING GIN (properties); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE TYPE item AS ( id text, geometry geometry, properties JSONB, collection_id text, datetime timestamptz ); CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ SELECT make_partitions(stac_datetime(data)); INSERT INTO items (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ DECLARE partition text; q text; newcontent jsonb; BEGIN PERFORM make_partitions(stac_datetime(data)); partition := get_partition(stac_datetime(data)); q := format($q$ INSERT INTO %I (content) VALUES ($1) ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content WHERE %I.content IS DISTINCT FROM EXCLUDED.content RETURNING content; $q$, partition, partition); EXECUTE q INTO newcontent USING (data); RAISE NOTICE 'newcontent: %', newcontent; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP RAISE NOTICE 'Analyzing %', p; EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION backfill_partitions() RETURNS VOID AS $$ DECLARE BEGIN IF EXISTS (SELECT 1 FROM items_default LIMIT 1) THEN RAISE NOTICE 'Creating new partitions and moving data from default'; CREATE TEMP TABLE items_default_tmp ON COMMIT DROP AS SELECT datetime, content FROM items_default; TRUNCATE items_default; PERFORM make_partitions(min(datetime), max(datetime)) FROM items_default_tmp; INSERT INTO items (content) SELECT content FROM items_default_tmp; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION items_trigger_stmt_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_stmt_trigger ON items; CREATE TRIGGER items_stmt_trigger AFTER INSERT OR UPDATE OR DELETE ON items FOR EACH STATEMENT EXECUTE PROCEDURE items_trigger_stmt_func(); /* View to get a table of available items partitions with date ranges */ --DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE OR REPLACE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; --DROP VIEW IF EXISTS items_partitions; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_by_partition( IN _where text DEFAULT 'TRUE', IN _dtrange tstzrange DEFAULT tstzrange('-infinity','infinity'), IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE partition_query text; main_query text; batchcount int; counter int := 0; p record; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange ASC ; $q$); ELSE partition_query := format($q$ SELECT 'items' as partition WHERE $1 IS NOT NULL; $q$); END IF; RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query USING (_dtrange) LOOP IF lower(_dtrange)::timestamptz > '-infinity' THEN _where := concat(_where,format(' AND datetime >= %L',lower(_dtrange)::timestamptz::text)); END IF; IF upper(_dtrange)::timestamptz < 'infinity' THEN _where := concat(_where,format(' AND datetime <= %L',upper(_dtrange)::timestamptz::text)); END IF; main_query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s LIMIT %s - $1 $q$, p.partition::text, _where, _orderby, _limit ); RAISE NOTICE 'Partition Query %', main_query; RAISE NOTICE '%', counter; RETURN QUERY EXECUTE main_query USING counter; GET DIAGNOSTICS batchcount = ROW_COUNT; counter := counter + batchcount; RAISE NOTICE 'FOUND %', batchcount; IF counter >= _limit THEN EXIT; END IF; RAISE NOTICE 'ADDED % FOR A TOTAL OF %', batchcount, counter; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION split_stac_path(IN path text, OUT col text, OUT dotpath text, OUT jspath text, OUT jspathtext text) AS $$ WITH col AS ( SELECT CASE WHEN split_part(path, '.', 1) IN ('id', 'stac_version', 'stac_extensions','geometry','properties','assets','collection_id','datetime','links', 'extra_fields') THEN split_part(path, '.', 1) ELSE 'properties' END AS col ), dp AS ( SELECT col, ltrim(replace(path, col , ''),'.') as dotpath FROM col ), paths AS ( SELECT col, dotpath, regexp_split_to_table(dotpath,E'\\.') as path FROM dp ) SELECT col, btrim(concat(col,'.',dotpath),'.'), CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, regexp_replace( CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, E'>([^>]*)$','>>\1' ) FROM paths group by col, dotpath; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions for searching items */ CREATE OR REPLACE FUNCTION sort_base( IN _sort jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]', OUT key text, OUT col text, OUT dir text, OUT rdir text, OUT sort text, OUT rsort text ) RETURNS SETOF RECORD AS $$ WITH sorts AS ( SELECT value->>'field' as key, (split_stac_path(value->>'field')).jspathtext as col, coalesce(upper(value->>'direction'),'ASC') as dir FROM jsonb_array_elements('[]'::jsonb || coalesce(_sort,'[{"field":"datetime","direction":"desc"}]') ) ) SELECT key, col, dir, CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END as rdir, concat(col, ' ', dir, ' NULLS LAST ') AS sort, concat(col,' ', CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END, ' NULLS LAST ') AS rsort FROM sorts UNION ALL SELECT 'id', 'id', 'DESC', 'ASC', 'id DESC', 'id ASC' ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(sort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION rsort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(rsort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION in_array_q(col text, arr jsonb) RETURNS text AS $$ SELECT CASE jsonb_typeof(arr) WHEN 'array' THEN format('%I = ANY(textarr(%L))', col, arr) ELSE format('%I = %L', col, arr) END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION count_by_delim(text, text) RETURNS int AS $$ SELECT count(*) FROM regexp_split_to_table($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$ DECLARE ret text := ''; op text; jp text; att_parts RECORD; val_str text; prop_path text; BEGIN val_str := lower(jsonb_build_object('a',val)->>'a'); RAISE NOTICE 'val_str %', val_str; att_parts := split_stac_path(att); prop_path := replace(att_parts.dotpath, 'properties.', ''); op := CASE _op WHEN 'eq' THEN '=' WHEN 'gte' THEN '>=' WHEN 'gt' THEN '>' WHEN 'lte' THEN '<=' WHEN 'lt' THEN '<' WHEN 'ne' THEN '!=' WHEN 'neq' THEN '!=' WHEN 'startsWith' THEN 'LIKE' WHEN 'endsWith' THEN 'LIKE' WHEN 'contains' THEN 'LIKE' ELSE _op END; val_str := CASE _op WHEN 'startsWith' THEN concat(val_str, '%') WHEN 'endsWith' THEN concat('%', val_str) WHEN 'contains' THEN concat('%',val_str,'%') ELSE val_str END; RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.'); IF op = '=' AND att_parts.col = 'properties' --AND count_by_delim(att_parts.dotpath,'\.') = 2 THEN -- use jsonpath query to leverage index for eqaulity tests on single level deep properties jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb)); raise notice 'jp: %', jp; ret := format($q$ properties @? %L $q$, jp); ELSIF jsonb_typeof(val) = 'number' THEN ret := format('properties ? %L AND (%s)::numeric %s %s', prop_path, att_parts.jspathtext, op, val); ELSE ret := format('properties ? %L AND %s %s %L', prop_path ,att_parts.jspathtext, op, val_str); END IF; RAISE NOTICE 'Op Query: %', ret; return ret; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION stac_query(_query jsonb) RETURNS TEXT[] AS $$ DECLARE qa text[]; att text; ops jsonb; op text; val jsonb; BEGIN FOR att, ops IN SELECT key, value FROM jsonb_each(_query) LOOP FOR op, val IN SELECT key, value FROM jsonb_each(ops) LOOP qa := array_append(qa, stac_query_op(att,op, val)); RAISE NOTICE '% % %', att, op, val; END LOOP; END LOOP; RETURN qa; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION filter_by_order(item_id text, _sort jsonb, _type text) RETURNS text AS $$ DECLARE item item; BEGIN SELECT * INTO item FROM items WHERE id=item_id; RETURN filter_by_order(item, _sort, _type); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; -- Used to create filters used for paging using the items id from the token CREATE OR REPLACE FUNCTION filter_by_order(_item item, _sort jsonb, _type text) RETURNS text AS $$ DECLARE sorts RECORD; filts text[]; itemval text; op text; idop text; ret text; eq_flag text; _item_j jsonb := to_jsonb(_item); BEGIN FOR sorts IN SELECT * FROM sort_base(_sort) LOOP IF sorts.col = 'datetime' THEN CONTINUE; END IF; IF sorts.col='id' AND _type IN ('prev','next') THEN eq_flag := ''; ELSE eq_flag := '='; END IF; op := concat( CASE WHEN _type in ('prev','first') AND sorts.dir = 'ASC' THEN '<' WHEN _type in ('last','next') AND sorts.dir = 'ASC' THEN '>' WHEN _type in ('prev','first') AND sorts.dir = 'DESC' THEN '>' WHEN _type in ('last','next') AND sorts.dir = 'DESC' THEN '<' END, eq_flag ); IF _item_j ? sorts.col THEN filts = array_append(filts, format('%s %s %L', sorts.col, op, _item_j->>sorts.col)); END IF; END LOOP; ret := coalesce(array_to_string(filts,' AND '), 'TRUE'); RAISE NOTICE 'Order Filter %', ret; RETURN ret; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION search_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ WITH t AS ( SELECT i, row_number() over () as r FROM jsonb_array_elements(j) i ), o AS ( SELECT i FROM t ORDER BY r DESC ) SELECT jsonb_agg(i) from o ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS SETOF jsonb AS $$ DECLARE qstart timestamptz := clock_timestamp(); _sort text := ''; _rsort text := ''; _limit int := 10; _geom geometry; qa text[]; pq text[]; query text; pq_prop record; pq_op record; prev_id text := NULL; next_id text := NULL; whereq text := 'TRUE'; links jsonb := '[]'::jsonb; token text; tok_val text; tok_q text := 'TRUE'; tok_sort text; first_id text; first_dt timestamptz; last_id text; sort text; rsort text; dt text[]; dqa text[]; dq text; mq_where text; startdt timestamptz; enddt timestamptz; item items%ROWTYPE; counter int := 0; batchcount int; month timestamptz; m record; _dtrange tstzrange := tstzrange('-infinity','infinity'); _dtsort text; _token_dtrange tstzrange := tstzrange('-infinity','infinity'); _token_record items%ROWTYPE; is_prev boolean := false; includes text[]; excludes text[]; BEGIN -- Create table from sort query of items to sort CREATE TEMP TABLE pgstac_tmp_sorts ON COMMIT DROP AS SELECT * FROM sort_base(_search->'sortby'); -- Get the datetime sort direction, necessary for efficient cycling through partitions SELECT INTO _dtsort dir FROM pgstac_tmp_sorts WHERE key='datetime'; RAISE NOTICE '_dtsort: %',_dtsort; SELECT INTO _sort string_agg(s.sort,', ') FROM pgstac_tmp_sorts s; SELECT INTO _rsort string_agg(s.rsort,', ') FROM pgstac_tmp_sorts s; tok_sort := _sort; -- Get datetime from query as a tstzrange IF _search ? 'datetime' THEN _dtrange := search_dtrange(_search->'datetime'); _token_dtrange := _dtrange; END IF; -- Get the paging token IF _search ? 'token' THEN token := _search->>'token'; tok_val := substr(token,6); IF starts_with(token, 'prev:') THEN is_prev := true; END IF; SELECT INTO _token_record * FROM items WHERE id=tok_val; IF (is_prev AND _dtsort = 'DESC') OR (not is_prev AND _dtsort = 'ASC') THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSIF _dtsort IS NOT NULL THEN _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; IF is_prev THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'first'); _sort := _rsort; ELSIF starts_with(token, 'next:') THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'last'); END IF; END IF; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'tok_q: % _token_dtrange: %', tok_q, _token_dtrange; IF _search ? 'ids' THEN RAISE NOTICE 'searching solely based on ids... %',_search; qa := array_append(qa, in_array_q('id', _search->'ids')); ELSE IF _search ? 'intersects' THEN _geom := ST_SetSRID(ST_GeomFromGeoJSON(_search->>'intersects'), 4326); ELSIF _search ? 'bbox' THEN _geom := bbox_geom(_search->'bbox'); END IF; IF _geom IS NOT NULL THEN qa := array_append(qa, format('st_intersects(geometry, %L::geometry)',_geom)); END IF; IF _search ? 'collections' THEN qa := array_append(qa, in_array_q('collection_id', _search->'collections')); END IF; IF _search ? 'query' THEN qa := array_cat(qa, stac_query(_search->'query') ); END IF; END IF; IF _search ? 'limit' THEN _limit := (_search->>'limit')::int; END IF; IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; RAISE NOTICE 'Includes: %, Excludes: %', includes, excludes; END IF; whereq := COALESCE(array_to_string(qa,' AND '),' TRUE '); dq := COALESCE(array_to_string(dqa,' AND '),' TRUE '); RAISE NOTICE 'timing before temp table: %', age(clock_timestamp(), qstart); CREATE TEMP TABLE results_page ON COMMIT DROP AS SELECT * FROM items_by_partition( concat(whereq, ' AND ', tok_q), _token_dtrange, _sort, _limit + 1 ); RAISE NOTICE 'timing after temp table: %', age(clock_timestamp(), qstart); RAISE NOTICE 'timing before min/max: %', age(clock_timestamp(), qstart); IF is_prev THEN SELECT INTO last_id, first_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; ELSE SELECT INTO first_id, last_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; END IF; RAISE NOTICE 'firstid: %, lastid %', first_id, last_id; RAISE NOTICE 'timing after min/max: %', age(clock_timestamp(), qstart); IF counter > _limit THEN next_id := last_id; RAISE NOTICE 'next_id: %', next_id; ELSE RAISE NOTICE 'No more next'; END IF; IF tok_q = 'TRUE' THEN RAISE NOTICE 'Not a paging query, no previous item'; ELSE RAISE NOTICE 'Getting previous item id'; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); SELECT INTO _token_record * FROM items WHERE id=first_id; IF _dtsort = 'DESC' THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSE _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; RAISE NOTICE '% %', _token_dtrange, _dtrange; SELECT id INTO prev_id FROM items_by_partition( concat(whereq, ' AND ', filter_by_order(first_id, _search->'sortby', 'prev')), _token_dtrange, _rsort, 1 ); RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'prev_id: %', prev_id; END IF; RETURN QUERY WITH features AS ( SELECT filter_jsonb(content, includes, excludes) as content FROM results_page LIMIT _limit ), j AS (SELECT jsonb_agg(content) as feature_arr FROM features) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce ( CASE WHEN is_prev THEN flip_jsonb_array(feature_arr) ELSE feature_arr END ,'[]'::jsonb), 'links', links, 'timeStamp', now(), 'next', next_id, 'prev', prev_id ) FROM j ; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; INSERT INTO pgstac.migrations (version) VALUES ('0.2.8'); ================================================ FILE: src/pgstac/migrations/pgstac.0.2.9-0.3.0.sql ================================================ SET SEARCH_PATH to pgstac, public; ALTER SCHEMA pgstac RENAME TO pgstac_bk; CREATE SCHEMA pgstac; ALTER TABLE pgstac_bk.migrations SET SCHEMA pgstac; ALTER TABLE pgstac_bk.collections SET SCHEMA pgstac; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange ASC ; $q$); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; --RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND %s ORDER BY %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where, _orderby ); --RAISE NOTICE 'query: %', query; RETURN NEXT query; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partion_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT DO NOTHING ; DELETE FROM items_staging_ignore; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging_upsert; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; ----- BEGIN SEARCH CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; -- format($F$ %s = %%s $F$, field); RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'id' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'id' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collection' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collection' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{id,collection,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "like": "%s LIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":"lower(%s)" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL 1: %', search; -- Convert any old style stac query to cql search := query_to_cqlfilter(search); RAISE NOTICE 'SEARCH CQL 2: %', search; -- Convert item,collection,datetime,bbox,intersects to cql search := add_filters_to_cql(search); RAISE NOTICE 'SEARCH CQL Final: %', search; _where := cql_query_op(search->'filter'); IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sorts AS ( SELECT (items_path(value->>'field')).path as key, parse_sort_dir(value->>'direction', reverse) as dir FROM jsonb_array_elements( '[]'::jsonb || coalesce(_search->'sort','[{"field":"datetime", "direction":"desc"}]') || '[{"field":"id","direction":"desc"}]'::jsonb ) ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sort','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id'); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb) RETURNS text AS $$ SELECT md5(search_tohash($1)::text); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, total_count bigint ); CREATE OR REPLACE FUNCTION search_query(_search jsonb = '{}'::jsonb, updatestats boolean DEFAULT false) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; BEGIN INSERT INTO searches (search) VALUES (search_tohash(_search)) ON CONFLICT DO NOTHING RETURNING * INTO search; IF search.hash IS NULL THEN SELECT * INTO search FROM searches WHERE hash=search_hash(_search); END IF; IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; IF search.statslastupdated IS NULL OR age(search.statslastupdated) > '1 day'::interval OR (_search ? 'context' AND search.total_count IS NULL) THEN updatestats := TRUE; END IF; IF updatestats THEN -- Get Estimated Stats RAISE NOTICE 'Getting stats for %', search._where; search.estimated_count := estimated_count(search._where); RAISE NOTICE 'Estimated Count: %', search.estimated_count; IF _search ? 'context' OR search.estimated_count < 10000 THEN --search.total_count := partition_count(search._where); EXECUTE format( 'SELECT count(*) FROM items WHERE %s', search._where ) INTO search.total_count; RAISE NOTICE 'Actual Count: %', search.total_count; ELSE search.total_count := NULL; END IF; search.statslastupdated := now(); END IF; search.lastused := now(); search.usecount := coalesce(search.usecount,0) + 1; RAISE NOTICE 'SEARCH: %', search; UPDATE searches SET _where = search._where, orderby = search.orderby, lastused = search.lastused, usecount = search.usecount, statslastupdated = search.statslastupdated, estimated_count = search.estimated_count, total_count = search.total_count WHERE hash = search.hash ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; total_count := coalesce(searches.total_count, searches.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby) LOOP timer := clock_timestamp(); query := format('%s LIMIT %L', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; curs = create_cursor(query); LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; RAISE NOTICE 'Query took %', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', out_records, 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off; ----- END SEARCH WITH t AS (SELECT items_partition_create_worker(partition, lower(tstzrange), upper(tstzrange)) FROM pgstac_bk.items_partitions) SELECT count(*) FROM t; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT id, geometry, collection_id, stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM pgstac_bk.items; DROP SCHEMA pgstac_bk CASCADE; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; INSERT INTO pgstac.migrations (version) VALUES ('0.3.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.2.9.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS partman; CREATE EXTENSION IF NOT EXISTS pg_partman SCHEMA partman; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT ARRAY(SELECT jsonb_array_elements_text(_js)); $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* converts a jsonb text array to comma delimited list of identifer quoted useful for constructing column lists for selects */ CREATE OR REPLACE FUNCTION array_idents(_js jsonb) RETURNS text AS $$ SELECT string_agg(quote_ident(v),',') FROM jsonb_array_elements_text(_js) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT (value->'properties'->>'datetime')::timestamptz; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION properties_idx(_in jsonb) RETURNS jsonb AS $$ WITH t AS ( select array_to_string(path,'.') as path, lower(value::text)::jsonb as lowerval FROM jsonb_val_paths(_in) WHERE array_to_string(path,'.') not in ('datetime') ) SELECT jsonb_object_agg(path, lowerval) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS items ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED NOT NULL, geometry geometry GENERATED ALWAYS AS (stac_geom(content)) STORED NOT NULL, properties jsonb GENERATED ALWAYS as (properties_idx(content->'properties')) STORED, collection_id text GENERATED ALWAYS AS (content->>'collection') STORED NOT NULL, datetime timestamptz GENERATED ALWAYS AS (stac_datetime(content)) STORED NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (stac_datetime(content)) ; ALTER TABLE items ADD constraint items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) DEFERRABLE; CREATE TABLE items_template ( LIKE items ); ALTER TABLE items_template ADD PRIMARY KEY (id); DELETE from partman.part_config WHERE parent_table = 'pgstac.items'; SELECT partman.create_parent( 'pgstac.items', 'datetime', 'native', 'weekly', p_template_table := 'pgstac.items_template', p_premake := 4 ); CREATE OR REPLACE FUNCTION make_partitions(st timestamptz, et timestamptz DEFAULT NULL) RETURNS BOOL AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', coalesce(et, st)), '1 week'::interval ) w ), w AS (SELECT array_agg(w) as w FROM t) SELECT CASE WHEN w IS NULL THEN NULL ELSE partman.create_partition_time('pgstac.items', w, true) END FROM w; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_partition(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE INDEX "datetime_id_idx" ON items (datetime, id); CREATE INDEX "properties_idx" ON items USING GIN (properties); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE TYPE item AS ( id text, geometry geometry, properties JSONB, collection_id text, datetime timestamptz ); CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ SELECT make_partitions(stac_datetime(data)); INSERT INTO items (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ DECLARE partition text; q text; newcontent jsonb; BEGIN PERFORM make_partitions(stac_datetime(data)); partition := get_partition(stac_datetime(data)); q := format($q$ INSERT INTO %I (content) VALUES ($1) ON CONFLICT (id) DO UPDATE SET content = EXCLUDED.content WHERE %I.content IS DISTINCT FROM EXCLUDED.content RETURNING content; $q$, partition, partition); EXECUTE q INTO newcontent USING (data); RAISE NOTICE 'newcontent: %', newcontent; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP RAISE NOTICE 'Analyzing %', p; EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION backfill_partitions() RETURNS VOID AS $$ DECLARE BEGIN IF EXISTS (SELECT 1 FROM items_default LIMIT 1) THEN RAISE NOTICE 'Creating new partitions and moving data from default'; CREATE TEMP TABLE items_default_tmp ON COMMIT DROP AS SELECT datetime, content FROM items_default; TRUNCATE items_default; PERFORM make_partitions(min(datetime), max(datetime)) FROM items_default_tmp; INSERT INTO items (content) SELECT content FROM items_default_tmp; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION items_trigger_stmt_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; DROP TRIGGER IF EXISTS items_stmt_trigger ON items; CREATE TRIGGER items_stmt_trigger AFTER INSERT OR UPDATE OR DELETE ON items FOR EACH STATEMENT EXECUTE PROCEDURE items_trigger_stmt_func(); /* View to get a table of available items partitions with date ranges */ --DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE OR REPLACE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; --DROP VIEW IF EXISTS items_partitions; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_by_partition( IN _where text DEFAULT 'TRUE', IN _dtrange tstzrange DEFAULT tstzrange('-infinity','infinity'), IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE partition_query text; main_query text; batchcount int; counter int := 0; p record; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition FROM items_partitions WHERE tstzrange && $1 ORDER BY tstzrange ASC ; $q$); ELSE partition_query := format($q$ SELECT 'items' as partition WHERE $1 IS NOT NULL; $q$); END IF; RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query USING (_dtrange) LOOP IF lower(_dtrange)::timestamptz > '-infinity' THEN _where := concat(_where,format(' AND datetime >= %L',lower(_dtrange)::timestamptz::text)); END IF; IF upper(_dtrange)::timestamptz < 'infinity' THEN _where := concat(_where,format(' AND datetime <= %L',upper(_dtrange)::timestamptz::text)); END IF; main_query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s LIMIT %s - $1 $q$, p.partition::text, _where, _orderby, _limit ); RAISE NOTICE 'Partition Query %', main_query; RAISE NOTICE '%', counter; RETURN QUERY EXECUTE main_query USING counter; GET DIAGNOSTICS batchcount = ROW_COUNT; counter := counter + batchcount; RAISE NOTICE 'FOUND %', batchcount; IF counter >= _limit THEN EXIT; END IF; RAISE NOTICE 'ADDED % FOR A TOTAL OF %', batchcount, counter; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION split_stac_path(IN path text, OUT col text, OUT dotpath text, OUT jspath text, OUT jspathtext text) AS $$ WITH col AS ( SELECT CASE WHEN split_part(path, '.', 1) IN ('id', 'stac_version', 'stac_extensions','geometry','properties','assets','collection_id','datetime','links', 'extra_fields') THEN split_part(path, '.', 1) ELSE 'properties' END AS col ), dp AS ( SELECT col, ltrim(replace(path, col , ''),'.') as dotpath FROM col ), paths AS ( SELECT col, dotpath, regexp_split_to_table(dotpath,E'\\.') as path FROM dp ) SELECT col, btrim(concat(col,'.',dotpath),'.'), CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, regexp_replace( CASE WHEN btrim(concat(col,'.',dotpath),'.') != col THEN concat(col,'->',string_agg(concat('''',path,''''),'->')) ELSE col END, E'>([^>]*)$','>>\1' ) FROM paths group by col, dotpath; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions for searching items */ CREATE OR REPLACE FUNCTION sort_base( IN _sort jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]', OUT key text, OUT col text, OUT dir text, OUT rdir text, OUT sort text, OUT rsort text ) RETURNS SETOF RECORD AS $$ WITH sorts AS ( SELECT value->>'field' as key, (split_stac_path(value->>'field')).jspathtext as col, coalesce(upper(value->>'direction'),'ASC') as dir FROM jsonb_array_elements('[]'::jsonb || coalesce(_sort,'[{"field":"datetime","direction":"desc"}]') ) ) SELECT key, col, dir, CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END as rdir, concat(col, ' ', dir, ' NULLS LAST ') AS sort, concat(col,' ', CASE dir WHEN 'DESC' THEN 'ASC' ELSE 'ASC' END, ' NULLS LAST ') AS rsort FROM sorts UNION ALL SELECT 'id', 'id', 'DESC', 'ASC', 'id DESC', 'id ASC' ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(sort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION rsort(_sort jsonb) RETURNS text AS $$ SELECT string_agg(rsort,', ') FROM sort_base(_sort); $$ LANGUAGE SQL PARALLEL SAFE SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION in_array_q(col text, arr jsonb) RETURNS text AS $$ SELECT CASE jsonb_typeof(arr) WHEN 'array' THEN format('%I = ANY(textarr(%L))', col, arr) ELSE format('%I = %L', col, arr) END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION count_by_delim(text, text) RETURNS int AS $$ SELECT count(*) FROM regexp_split_to_table($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_query_op(att text, _op text, val jsonb) RETURNS text AS $$ DECLARE ret text := ''; op text; jp text; att_parts RECORD; val_str text; prop_path text; BEGIN val_str := lower(jsonb_build_object('a',val)->>'a'); RAISE NOTICE 'val_str %', val_str; att_parts := split_stac_path(att); prop_path := replace(att_parts.dotpath, 'properties.', ''); op := CASE _op WHEN 'eq' THEN '=' WHEN 'gte' THEN '>=' WHEN 'gt' THEN '>' WHEN 'lte' THEN '<=' WHEN 'lt' THEN '<' WHEN 'ne' THEN '!=' WHEN 'neq' THEN '!=' WHEN 'startsWith' THEN 'LIKE' WHEN 'endsWith' THEN 'LIKE' WHEN 'contains' THEN 'LIKE' ELSE _op END; val_str := CASE _op WHEN 'startsWith' THEN concat(val_str, '%') WHEN 'endsWith' THEN concat('%', val_str) WHEN 'contains' THEN concat('%',val_str,'%') ELSE val_str END; RAISE NOTICE 'att_parts: % %', att_parts, count_by_delim(att_parts.dotpath,'\.'); IF op = '=' AND att_parts.col = 'properties' --AND count_by_delim(att_parts.dotpath,'\.') = 2 THEN -- use jsonpath query to leverage index for eqaulity tests on single level deep properties jp := btrim(format($jp$ $.%I[*] ? ( @ == %s ) $jp$, replace(att_parts.dotpath, 'properties.',''), lower(val::text)::jsonb)); raise notice 'jp: %', jp; ret := format($q$ properties @? %L $q$, jp); ELSIF jsonb_typeof(val) = 'number' THEN ret := format('properties ? %L AND (%s)::numeric %s %s', prop_path, att_parts.jspathtext, op, val); ELSE ret := format('properties ? %L AND %s %s %L', prop_path ,att_parts.jspathtext, op, val_str); END IF; RAISE NOTICE 'Op Query: %', ret; return ret; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION stac_query(_query jsonb) RETURNS TEXT[] AS $$ DECLARE qa text[]; att text; ops jsonb; op text; val jsonb; BEGIN FOR att, ops IN SELECT key, value FROM jsonb_each(_query) LOOP FOR op, val IN SELECT key, value FROM jsonb_each(ops) LOOP qa := array_append(qa, stac_query_op(att,op, val)); RAISE NOTICE '% % %', att, op, val; END LOOP; END LOOP; RETURN qa; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION filter_by_order(item_id text, _sort jsonb, _type text) RETURNS text AS $$ DECLARE item item; BEGIN SELECT * INTO item FROM items WHERE id=item_id; RETURN filter_by_order(item, _sort, _type); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; -- Used to create filters used for paging using the items id from the token CREATE OR REPLACE FUNCTION filter_by_order(_item item, _sort jsonb, _type text) RETURNS text AS $$ DECLARE sorts RECORD; filts text[]; itemval text; op text; idop text; ret text; eq_flag text; _item_j jsonb := to_jsonb(_item); BEGIN FOR sorts IN SELECT * FROM sort_base(_sort) LOOP IF sorts.col = 'datetime' THEN CONTINUE; END IF; IF sorts.col='id' AND _type IN ('prev','next') THEN eq_flag := ''; ELSE eq_flag := '='; END IF; op := concat( CASE WHEN _type in ('prev','first') AND sorts.dir = 'ASC' THEN '<' WHEN _type in ('last','next') AND sorts.dir = 'ASC' THEN '>' WHEN _type in ('prev','first') AND sorts.dir = 'DESC' THEN '>' WHEN _type in ('last','next') AND sorts.dir = 'DESC' THEN '<' END, eq_flag ); IF _item_j ? sorts.col THEN filts = array_append(filts, format('%s %s %L', sorts.col, op, _item_j->>sorts.col)); END IF; END LOOP; ret := coalesce(array_to_string(filts,' AND '), 'TRUE'); RAISE NOTICE 'Order Filter %', ret; RETURN ret; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION search_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ WITH t AS ( SELECT i, row_number() over () as r FROM jsonb_array_elements(j) i ), o AS ( SELECT i FROM t ORDER BY r DESC ) SELECT jsonb_agg(i) from o ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS SETOF jsonb AS $$ DECLARE qstart timestamptz := clock_timestamp(); _sort text := ''; _rsort text := ''; _limit int := 10; _geom geometry; qa text[]; pq text[]; query text; pq_prop record; pq_op record; prev_id text := NULL; next_id text := NULL; whereq text := 'TRUE'; links jsonb := '[]'::jsonb; token text; tok_val text; tok_q text := 'TRUE'; tok_sort text; first_id text; first_dt timestamptz; last_id text; sort text; rsort text; dt text[]; dqa text[]; dq text; mq_where text; startdt timestamptz; enddt timestamptz; item items%ROWTYPE; counter int := 0; batchcount int; month timestamptz; m record; _dtrange tstzrange := tstzrange('-infinity','infinity'); _dtsort text; _token_dtrange tstzrange := tstzrange('-infinity','infinity'); _token_record items%ROWTYPE; is_prev boolean := false; includes text[]; excludes text[]; BEGIN -- Create table from sort query of items to sort CREATE TEMP TABLE pgstac_tmp_sorts ON COMMIT DROP AS SELECT * FROM sort_base(_search->'sortby'); -- Get the datetime sort direction, necessary for efficient cycling through partitions SELECT INTO _dtsort dir FROM pgstac_tmp_sorts WHERE key='datetime'; RAISE NOTICE '_dtsort: %',_dtsort; SELECT INTO _sort string_agg(s.sort,', ') FROM pgstac_tmp_sorts s; SELECT INTO _rsort string_agg(s.rsort,', ') FROM pgstac_tmp_sorts s; tok_sort := _sort; -- Get datetime from query as a tstzrange IF _search ? 'datetime' THEN _dtrange := search_dtrange(_search->'datetime'); _token_dtrange := _dtrange; END IF; -- Get the paging token IF _search ? 'token' THEN token := _search->>'token'; tok_val := substr(token,6); IF starts_with(token, 'prev:') THEN is_prev := true; END IF; SELECT INTO _token_record * FROM items WHERE id=tok_val; IF (is_prev AND _dtsort = 'DESC') OR (not is_prev AND _dtsort = 'ASC') THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSIF _dtsort IS NOT NULL THEN _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; IF is_prev THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'first'); _sort := _rsort; ELSIF starts_with(token, 'next:') THEN tok_q := filter_by_order(tok_val, _search->'sortby', 'last'); END IF; END IF; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'tok_q: % _token_dtrange: %', tok_q, _token_dtrange; IF _search ? 'ids' THEN RAISE NOTICE 'searching solely based on ids... %',_search; qa := array_append(qa, in_array_q('id', _search->'ids')); ELSE IF _search ? 'intersects' THEN _geom := ST_SetSRID(ST_GeomFromGeoJSON(_search->>'intersects'), 4326); ELSIF _search ? 'bbox' THEN _geom := bbox_geom(_search->'bbox'); END IF; IF _geom IS NOT NULL THEN qa := array_append(qa, format('st_intersects(geometry, %L::geometry)',_geom)); END IF; IF _search ? 'collections' THEN qa := array_append(qa, in_array_q('collection_id', _search->'collections')); END IF; IF _search ? 'query' THEN qa := array_cat(qa, stac_query(_search->'query') ); END IF; END IF; IF _search ? 'limit' THEN _limit := (_search->>'limit')::int; END IF; IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; RAISE NOTICE 'Includes: %, Excludes: %', includes, excludes; END IF; whereq := COALESCE(array_to_string(qa,' AND '),' TRUE '); dq := COALESCE(array_to_string(dqa,' AND '),' TRUE '); RAISE NOTICE 'timing before temp table: %', age(clock_timestamp(), qstart); CREATE TEMP TABLE results_page ON COMMIT DROP AS SELECT * FROM items_by_partition( concat(whereq, ' AND ', tok_q), _token_dtrange, _sort, _limit + 1 ); RAISE NOTICE 'timing after temp table: %', age(clock_timestamp(), qstart); RAISE NOTICE 'timing before min/max: %', age(clock_timestamp(), qstart); IF is_prev THEN SELECT INTO last_id, first_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; ELSE SELECT INTO first_id, last_id, counter first_value(id) OVER (), last_value(id) OVER (), count(*) OVER () FROM results_page; END IF; RAISE NOTICE 'firstid: %, lastid %', first_id, last_id; RAISE NOTICE 'timing after min/max: %', age(clock_timestamp(), qstart); IF counter > _limit THEN next_id := last_id; RAISE NOTICE 'next_id: %', next_id; ELSE RAISE NOTICE 'No more next'; END IF; IF tok_q = 'TRUE' THEN RAISE NOTICE 'Not a paging query, no previous item'; ELSE RAISE NOTICE 'Getting previous item id'; RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); SELECT INTO _token_record * FROM items WHERE id=first_id; IF _dtsort = 'DESC' THEN _token_dtrange := _dtrange * tstzrange(_token_record.datetime, 'infinity'); ELSE _token_dtrange := _dtrange * tstzrange('-infinity',_token_record.datetime); END IF; RAISE NOTICE '% %', _token_dtrange, _dtrange; SELECT id INTO prev_id FROM items_by_partition( concat(whereq, ' AND ', filter_by_order(first_id, _search->'sortby', 'prev')), _token_dtrange, _rsort, 1 ); RAISE NOTICE 'timing: %', age(clock_timestamp(), qstart); RAISE NOTICE 'prev_id: %', prev_id; END IF; RETURN QUERY WITH features AS ( SELECT filter_jsonb(content, includes, excludes) as content FROM results_page LIMIT _limit ), j AS (SELECT jsonb_agg(content) as feature_arr FROM features) SELECT jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce ( CASE WHEN is_prev THEN flip_jsonb_array(feature_arr) ELSE feature_arr END ,'[]'::jsonb), 'links', links, 'timeStamp', now(), 'next', next_id, 'prev', prev_id ) FROM j ; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; INSERT INTO pgstac.migrations (version) VALUES ('0.2.9'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.0-0.3.1.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.sort_sqlorderby(_search jsonb DEFAULT NULL::jsonb, reverse boolean DEFAULT false) RETURNS text LANGUAGE sql AS $function$ WITH sorts AS ( SELECT (items_path(value->>'field')).path as key, parse_sort_dir(value->>'direction', reverse) as dir FROM jsonb_array_elements( '[]'::jsonb || coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') || '[{"field":"id","direction":"desc"}]'::jsonb ) ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $function$ ; INSERT INTO migrations (version) VALUES ('0.3.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.0.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange ASC ; $q$); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND %s ORDER BY %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where, _orderby ); RETURN NEXT query; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partion_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT DO NOTHING ; DELETE FROM items_staging_ignore; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging_upsert; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; -- format($F$ %s = %%s $F$, field); RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'id' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'id' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collection' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collection' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{id,collection,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "like": "%s LIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":"lower(%s)" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL 1: %', search; -- Convert any old style stac query to cql search := query_to_cqlfilter(search); RAISE NOTICE 'SEARCH CQL 2: %', search; -- Convert item,collection,datetime,bbox,intersects to cql search := add_filters_to_cql(search); RAISE NOTICE 'SEARCH CQL Final: %', search; _where := cql_query_op(search->'filter'); IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sorts AS ( SELECT (items_path(value->>'field')).path as key, parse_sort_dir(value->>'direction', reverse) as dir FROM jsonb_array_elements( '[]'::jsonb || coalesce(_search->'sort','[{"field":"datetime", "direction":"desc"}]') || '[{"field":"id","direction":"desc"}]'::jsonb ) ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sort','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id'); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb) RETURNS text AS $$ SELECT md5(search_tohash($1)::text); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, total_count bigint ); CREATE OR REPLACE FUNCTION search_query(_search jsonb = '{}'::jsonb, updatestats boolean DEFAULT false) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; BEGIN INSERT INTO searches (search) VALUES (search_tohash(_search)) ON CONFLICT DO NOTHING RETURNING * INTO search; IF search.hash IS NULL THEN SELECT * INTO search FROM searches WHERE hash=search_hash(_search); END IF; IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; IF search.statslastupdated IS NULL OR age(search.statslastupdated) > '1 day'::interval OR (_search ? 'context' AND search.total_count IS NULL) THEN updatestats := TRUE; END IF; IF updatestats THEN -- Get Estimated Stats RAISE NOTICE 'Getting stats for %', search._where; search.estimated_count := estimated_count(search._where); RAISE NOTICE 'Estimated Count: %', search.estimated_count; IF _search ? 'context' OR search.estimated_count < 10000 THEN --search.total_count := partition_count(search._where); EXECUTE format( 'SELECT count(*) FROM items WHERE %s', search._where ) INTO search.total_count; RAISE NOTICE 'Actual Count: %', search.total_count; ELSE search.total_count := NULL; END IF; search.statslastupdated := now(); END IF; search.lastused := now(); search.usecount := coalesce(search.usecount,0) + 1; RAISE NOTICE 'SEARCH: %', search; UPDATE searches SET _where = search._where, orderby = search.orderby, lastused = search.lastused, usecount = search.usecount, statslastupdated = search.statslastupdated, estimated_count = search.estimated_count, total_count = search.total_count WHERE hash = search.hash ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; total_count := coalesce(searches.total_count, searches.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby) LOOP timer := clock_timestamp(); query := format('%s LIMIT %L', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; curs = create_cursor(query); LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; RAISE NOTICE 'Query took %', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', out_records, 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off; INSERT INTO pgstac.migrations (version) VALUES ('0.3.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.1-0.3.2.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.add_filters_to_cql(j jsonb) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'id' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'id' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{id,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $function$ ; INSERT INTO migrations (version) VALUES ('0.3.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.1.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange ASC ; $q$); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND %s ORDER BY %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where, _orderby ); RETURN NEXT query; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partion_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT DO NOTHING ; DELETE FROM items_staging_ignore; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging_upsert; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; -- format($F$ %s = %%s $F$, field); RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'id' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'id' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collection' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collection' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{id,collection,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "like": "%s LIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":"lower(%s)" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL 1: %', search; -- Convert any old style stac query to cql search := query_to_cqlfilter(search); RAISE NOTICE 'SEARCH CQL 2: %', search; -- Convert item,collection,datetime,bbox,intersects to cql search := add_filters_to_cql(search); RAISE NOTICE 'SEARCH CQL Final: %', search; _where := cql_query_op(search->'filter'); IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sorts AS ( SELECT (items_path(value->>'field')).path as key, parse_sort_dir(value->>'direction', reverse) as dir FROM jsonb_array_elements( '[]'::jsonb || coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') || '[{"field":"id","direction":"desc"}]'::jsonb ) ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sort','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id'); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb) RETURNS text AS $$ SELECT md5(search_tohash($1)::text); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, total_count bigint ); CREATE OR REPLACE FUNCTION search_query(_search jsonb = '{}'::jsonb, updatestats boolean DEFAULT false) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; BEGIN INSERT INTO searches (search) VALUES (search_tohash(_search)) ON CONFLICT DO NOTHING RETURNING * INTO search; IF search.hash IS NULL THEN SELECT * INTO search FROM searches WHERE hash=search_hash(_search); END IF; IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; IF search.statslastupdated IS NULL OR age(search.statslastupdated) > '1 day'::interval OR (_search ? 'context' AND search.total_count IS NULL) THEN updatestats := TRUE; END IF; IF updatestats THEN -- Get Estimated Stats RAISE NOTICE 'Getting stats for %', search._where; search.estimated_count := estimated_count(search._where); RAISE NOTICE 'Estimated Count: %', search.estimated_count; IF _search ? 'context' OR search.estimated_count < 10000 THEN --search.total_count := partition_count(search._where); EXECUTE format( 'SELECT count(*) FROM items WHERE %s', search._where ) INTO search.total_count; RAISE NOTICE 'Actual Count: %', search.total_count; ELSE search.total_count := NULL; END IF; search.statslastupdated := now(); END IF; search.lastused := now(); search.usecount := coalesce(search.usecount,0) + 1; RAISE NOTICE 'SEARCH: %', search; UPDATE searches SET _where = search._where, orderby = search.orderby, lastused = search.lastused, usecount = search.usecount, statslastupdated = search.statslastupdated, estimated_count = search.estimated_count, total_count = search.total_count WHERE hash = search.hash ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; total_count := coalesce(searches.total_count, searches.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby) LOOP timer := clock_timestamp(); query := format('%s LIMIT %L', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; curs = create_cursor(query); LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; RAISE NOTICE 'Query took %', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', out_records, 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off; INSERT INTO pgstac.migrations (version) VALUES ('0.3.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.2-0.3.3.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.add_filters_to_cql(j jsonb) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_token_filter(_search jsonb DEFAULT '{}'::jsonb, token_rec jsonb DEFAULT NULL::jsonb) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SET jit TO 'off' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; total_count := coalesce(searches.total_count, searches.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby) LOOP timer := clock_timestamp(); query := format('%s LIMIT %L', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; curs = create_cursor(query); LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; RAISE NOTICE 'Query took %', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.sort_sqlorderby(_search jsonb DEFAULT NULL::jsonb, reverse boolean DEFAULT false) RETURNS text LANGUAGE sql AS $function$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT (items_path(value->>'field')).path as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $function$ ; INSERT INTO migrations (version) VALUES ('0.3.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.2.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange ASC ; $q$); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND %s ORDER BY %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where, _orderby ); RETURN NEXT query; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partion_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT DO NOTHING ; DELETE FROM items_staging_ignore; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging_upsert; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; -- format($F$ %s = %%s $F$, field); RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'id' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'id' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{id,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "like": "%s LIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":"lower(%s)" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL 1: %', search; -- Convert any old style stac query to cql search := query_to_cqlfilter(search); RAISE NOTICE 'SEARCH CQL 2: %', search; -- Convert item,collection,datetime,bbox,intersects to cql search := add_filters_to_cql(search); RAISE NOTICE 'SEARCH CQL Final: %', search; _where := cql_query_op(search->'filter'); IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sorts AS ( SELECT (items_path(value->>'field')).path as key, parse_sort_dir(value->>'direction', reverse) as dir FROM jsonb_array_elements( '[]'::jsonb || coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') || '[{"field":"id","direction":"desc"}]'::jsonb ) ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sort','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id'); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb) RETURNS text AS $$ SELECT md5(search_tohash($1)::text); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, total_count bigint ); CREATE OR REPLACE FUNCTION search_query(_search jsonb = '{}'::jsonb, updatestats boolean DEFAULT false) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; BEGIN INSERT INTO searches (search) VALUES (search_tohash(_search)) ON CONFLICT DO NOTHING RETURNING * INTO search; IF search.hash IS NULL THEN SELECT * INTO search FROM searches WHERE hash=search_hash(_search); END IF; IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; IF search.statslastupdated IS NULL OR age(search.statslastupdated) > '1 day'::interval OR (_search ? 'context' AND search.total_count IS NULL) THEN updatestats := TRUE; END IF; IF updatestats THEN -- Get Estimated Stats RAISE NOTICE 'Getting stats for %', search._where; search.estimated_count := estimated_count(search._where); RAISE NOTICE 'Estimated Count: %', search.estimated_count; IF _search ? 'context' OR search.estimated_count < 10000 THEN --search.total_count := partition_count(search._where); EXECUTE format( 'SELECT count(*) FROM items WHERE %s', search._where ) INTO search.total_count; RAISE NOTICE 'Actual Count: %', search.total_count; ELSE search.total_count := NULL; END IF; search.statslastupdated := now(); END IF; search.lastused := now(); search.usecount := coalesce(search.usecount,0) + 1; RAISE NOTICE 'SEARCH: %', search; UPDATE searches SET _where = search._where, orderby = search.orderby, lastused = search.lastused, usecount = search.usecount, statslastupdated = search.statslastupdated, estimated_count = search.estimated_count, total_count = search.total_count WHERE hash = search.hash ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; total_count := coalesce(searches.total_count, searches.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby) LOOP timer := clock_timestamp(); query := format('%s LIMIT %L', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; curs = create_cursor(query); LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; RAISE NOTICE 'Query took %', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', out_records, 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off; INSERT INTO pgstac.migrations (version) VALUES ('0.3.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.3-0.3.4.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.create_items(data jsonb) RETURNS void LANGUAGE sql SET search_path TO 'pgstac', 'public' AS $function$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $function$ ; CREATE OR REPLACE FUNCTION pgstac.ftime() RETURNS interval LANGUAGE sql AS $function$ SELECT age(clock_timestamp(), transaction_timestamp()); $function$ ; CREATE OR REPLACE FUNCTION pgstac.geojsonsearch(geojson jsonb, queryhash text, fields jsonb DEFAULT NULL::jsonb, _scanlimit integer DEFAULT 10000, _limit integer DEFAULT 100, _timelimit interval DEFAULT '00:00:05'::interval, exitwhenfull boolean DEFAULT true, skipcovered boolean DEFAULT true) RETURNS jsonb LANGUAGE sql AS $function$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $function$ ; CREATE OR REPLACE FUNCTION pgstac.geometrysearch(geom geometry, queryhash text, fields jsonb DEFAULT NULL::jsonb, _scanlimit integer DEFAULT 10000, _limit integer DEFAULT 100, _timelimit interval DEFAULT '00:00:05'::interval, exitwhenfull boolean DEFAULT true, skipcovered boolean DEFAULT true) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb[] := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); IF fields IS NOT NULL THEN IF fields ? 'fields' THEN fields := fields->'fields'; END IF; IF fields ? 'exclude' THEN excludes=textarr(fields->'exclude'); END IF; IF fields ? 'include' THEN includes=textarr(fields->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; END IF; RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; curs = create_cursor(query); LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; IF fields IS NOT NULL THEN out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); ELSE out_records := out_records || iter_record.content; END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', array_to_json(out_records)::jsonb ); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.tileenvelope(zoom integer, x integer, y integer) RETURNS geometry LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $function$ ; CREATE OR REPLACE FUNCTION pgstac.upsert_items(data jsonb) RETURNS void LANGUAGE sql SET search_path TO 'pgstac', 'public' AS $function$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $function$ ; CREATE OR REPLACE FUNCTION pgstac.xyzsearch(_x integer, _y integer, _z integer, queryhash text, fields jsonb DEFAULT NULL::jsonb, _scanlimit integer DEFAULT 10000, _limit integer DEFAULT 100, _timelimit interval DEFAULT '00:00:05'::interval, exitwhenfull boolean DEFAULT true, skipcovered boolean DEFAULT true) RETURNS jsonb LANGUAGE sql AS $function$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $function$ ; INSERT INTO migrations (version) VALUES ('0.3.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.3.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange ASC ; $q$); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND %s ORDER BY %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where, _orderby ); RETURN NEXT query; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partion_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT DO NOTHING ; DELETE FROM items_staging_ignore; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging_upsert; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; -- format($F$ %s = %%s $F$, field); RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "like": "%s LIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":"lower(%s)" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL 1: %', search; -- Convert any old style stac query to cql search := query_to_cqlfilter(search); RAISE NOTICE 'SEARCH CQL 2: %', search; -- Convert item,collection,datetime,bbox,intersects to cql search := add_filters_to_cql(search); RAISE NOTICE 'SEARCH CQL Final: %', search; _where := cql_query_op(search->'filter'); IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT (items_path(value->>'field')).path as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb) RETURNS text AS $$ SELECT md5(search_tohash($1)::text); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, total_count bigint ); CREATE OR REPLACE FUNCTION search_query(_search jsonb = '{}'::jsonb, updatestats boolean DEFAULT false) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; BEGIN INSERT INTO searches (search) VALUES (search_tohash(_search)) ON CONFLICT DO NOTHING RETURNING * INTO search; IF search.hash IS NULL THEN SELECT * INTO search FROM searches WHERE hash=search_hash(_search); END IF; IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; IF search.statslastupdated IS NULL OR age(search.statslastupdated) > '1 day'::interval OR (_search ? 'context' AND search.total_count IS NULL) THEN updatestats := TRUE; END IF; IF updatestats THEN -- Get Estimated Stats RAISE NOTICE 'Getting stats for %', search._where; search.estimated_count := estimated_count(search._where); RAISE NOTICE 'Estimated Count: %', search.estimated_count; IF _search ? 'context' OR search.estimated_count < 10000 THEN --search.total_count := partition_count(search._where); EXECUTE format( 'SELECT count(*) FROM items WHERE %s', search._where ) INTO search.total_count; RAISE NOTICE 'Actual Count: %', search.total_count; ELSE search.total_count := NULL; END IF; search.statslastupdated := now(); END IF; search.lastused := now(); search.usecount := coalesce(search.usecount,0) + 1; RAISE NOTICE 'SEARCH: %', search; UPDATE searches SET _where = search._where, orderby = search.orderby, lastused = search.lastused, usecount = search.usecount, statslastupdated = search.statslastupdated, estimated_count = search.estimated_count, total_count = search.total_count WHERE hash = search.hash ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; total_count := coalesce(searches.total_count, searches.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby) LOOP timer := clock_timestamp(); query := format('%s LIMIT %L', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; curs = create_cursor(query); LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; RAISE NOTICE 'Query took %', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off; INSERT INTO pgstac.migrations (version) VALUES ('0.3.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.4-0.3.5.sql ================================================ SET SEARCH_PATH to pgstac, public; CREATE TEMP TABLE temp_migrations AS SELECT version, max(datetime) as datetime from migrations group by 1; TRUNCATE pgstac.migrations; INSERT INTO pgstac.migrations SELECT * FROM temp_migrations; drop function if exists "pgstac"."partition_queries"(_where text, _orderby text); --drop function if exists "pgstac"."search_hash"(jsonb); drop function if exists "pgstac"."search_query"(_search jsonb, updatestats boolean); drop view if exists "pgstac"."items_partitions"; drop view if exists "pgstac"."all_items_partitions"; create table "pgstac"."search_wheres" ( "_where" text not null, "lastused" timestamp with time zone default now(), "usecount" bigint default 0, "statslastupdated" timestamp with time zone, "estimated_count" bigint, "estimated_cost" double precision, "time_to_estimate" double precision, "total_count" bigint, "time_to_count" double precision, "partitions" text[] ); alter table "pgstac"."migrations" alter column "datetime" set default clock_timestamp(); alter table "pgstac"."migrations" alter column "version" set not null; alter table "pgstac"."searches" drop column "estimated_count"; alter table "pgstac"."searches" drop column "statslastupdated"; alter table "pgstac"."searches" drop column "total_count"; alter table "pgstac"."searches" add column "metadata" jsonb not null default '{}'::jsonb; --alter table "pgstac"."searches" alter column "hash" set default pgstac.search_hash(search, metadata); CREATE OR REPLACE FUNCTION pgstac.search_hash(jsonb, jsonb) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $function$ ; alter table "pgstac"."searches" DROP COLUMN hash; alter table "pgstac"."searches" add column "hash" text generated always as ("pgstac".search_hash(search, metadata)) stored primary key; CREATE UNIQUE INDEX migrations_pkey ON pgstac.migrations USING btree (version); CREATE UNIQUE INDEX search_wheres_pkey ON pgstac.search_wheres USING btree (_where); alter table "pgstac"."migrations" add constraint "migrations_pkey" PRIMARY KEY using index "migrations_pkey"; alter table "pgstac"."search_wheres" add constraint "search_wheres_pkey" PRIMARY KEY using index "search_wheres_pkey"; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.array_reverse(anyarray) RETURNS anyarray LANGUAGE sql IMMUTABLE STRICT AS $function$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $function$ ; CREATE OR REPLACE FUNCTION pgstac.context() RETURNS text LANGUAGE sql AS $function$ SELECT get_setting('pgstac.context','off'::text); $function$ ; CREATE OR REPLACE FUNCTION pgstac.context_estimated_cost() RETURNS double precision LANGUAGE sql AS $function$ SELECT get_setting('pgstac.context_estimated_cost', 1000000::float); $function$ ; CREATE OR REPLACE FUNCTION pgstac.context_estimated_count() RETURNS integer LANGUAGE sql AS $function$ SELECT get_setting('pgstac.context_estimated_count', 100000::int); $function$ ; CREATE OR REPLACE FUNCTION pgstac.context_stats_ttl() RETURNS interval LANGUAGE sql AS $function$ SELECT get_setting('pgstac.context_stats_ttl', '1 day'::interval); $function$ ; CREATE OR REPLACE FUNCTION pgstac.drop_partition_constraints(partition text) RETURNS void LANGUAGE plpgsql AS $function$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, partition, end_datetime_constraint, partition, collections_constraint ); EXECUTE q; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.field_orderby(p text) RETURNS text LANGUAGE sql AS $function$ WITH t AS ( SELECT replace(trim(substring(indexdef from 'btree \((.*)\)')),' ','')as s FROM pg_indexes WHERE schemaname='pgstac' AND tablename='items' AND indexdef ~* 'btree' AND indexdef ~* 'properties' ) SELECT s FROM t WHERE strpos(s, lower(trim(p)))>0; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_setting(setting text, INOUT _default anynonarray DEFAULT NULL::text) RETURNS anynonarray LANGUAGE plpgsql AS $function$ DECLARE _type text; BEGIN SELECT pg_typeof(_default) INTO _type; IF _type = 'unknown' THEN _type='text'; END IF; EXECUTE format($q$ SELECT COALESCE( CAST(current_setting($1,TRUE) AS %s), $2 ) $q$, _type) INTO _default USING setting, _default ; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_version() RETURNS text LANGUAGE sql SET search_path TO 'pgstac', 'public' AS $function$ SELECT version FROM migrations ORDER BY datetime DESC, version DESC LIMIT 1; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_count(_where text) RETURNS bigint LANGUAGE plpgsql AS $function$ DECLARE cnt bigint; BEGIN EXECUTE format('SELECT count(*) FROM items WHERE %s', _where) INTO cnt; RETURN cnt; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.partition_checks(partition text, OUT min_datetime timestamp with time zone, OUT max_datetime timestamp with time zone, OUT min_end_datetime timestamp with time zone, OUT max_end_datetime timestamp with time zone, OUT collections text[], OUT cnt bigint) RETURNS record LANGUAGE plpgsql AS $function$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; q := format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime), array_agg(DISTINCT collection_id), count(*) FROM %I; $q$, partition ); EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; RAISE NOTICE '% % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; IF cnt IS NULL or cnt = 0 THEN RAISE NOTICE 'Partition % is empty, removing...', partition; q := format($q$ DROP TABLE IF EXISTS %I; $q$, partition ); EXECUTE q; RETURN; END IF; q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I check((end_datetime >= %L) AND (end_datetime <= %L)); ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I check((collection_id = ANY(%L))); ANALYZE %I; $q$, partition, end_datetime_constraint, partition, end_datetime_constraint, min_end_datetime, max_end_datetime, partition, collections_constraint, partition, collections_constraint, collections, partition ); EXECUTE q; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.partition_queries(_where text DEFAULT 'TRUE'::text, _orderby text DEFAULT 'datetime DESC, id DESC'::text, partitions text[] DEFAULT '{items}'::text[]) RETURNS SETOF text LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' AS $function$ DECLARE partition_query text; query text; p text; cursors refcursor; dstart timestamptz; dend timestamptz; step interval := '10 weeks'::interval; BEGIN IF _orderby ILIKE 'datetime d%' THEN partitions := partitions; ELSIF _orderby ILIKE 'datetime a%' THEN partitions := array_reverse(partitions); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RAISE NOTICE 'PARTITIONS ---> %',partitions; IF cardinality(partitions) > 0 THEN FOREACH p IN ARRAY partitions --EXECUTE partition_query LOOP query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s $q$, p, _where, _orderby ); RETURN NEXT query; END LOOP; END IF; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) RETURNS pgstac.searches LANGUAGE plpgsql AS $function$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.set_version(text) RETURNS text LANGUAGE sql SET search_path TO 'pgstac', 'public' AS $function$ INSERT INTO migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $function$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false) RETURNS pgstac.search_wheres LANGUAGE plpgsql AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl() OR (context() != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context() = 'on' OR ( context() = 'auto' AND ( sw.estimated_count < context_estimated_count() OR sw.estimated_cost < context_estimated_cost() ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres SELECT sw.* ON CONFLICT (_where) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $function$ ; create or replace view "pgstac"."all_items_partitions" as WITH base AS ( SELECT ((c.oid)::regclass)::text AS partition, pg_get_expr(c.relpartbound, c.oid) AS _constraint, regexp_matches(pg_get_expr(c.relpartbound, c.oid), '\(''([0-9 :+-]*)''\).*\(''([0-9 :+-]*)''\)'::text) AS t, (c.reltuples)::bigint AS est_cnt FROM pg_class c, pg_inherits i WHERE ((c.oid = i.inhrelid) AND (i.inhparent = ('pgstac.items'::regclass)::oid)) ) SELECT base.partition, tstzrange((base.t[1])::timestamp with time zone, (base.t[2])::timestamp with time zone) AS tstzrange, (base.t[1])::timestamp with time zone AS pstart, (base.t[2])::timestamp with time zone AS pend, base.est_cnt FROM base ORDER BY (tstzrange((base.t[1])::timestamp with time zone, (base.t[2])::timestamp with time zone)) DESC; CREATE OR REPLACE FUNCTION pgstac.items_partition_name(timestamp with time zone) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $function$ ; create or replace view "pgstac"."items_partitions" as SELECT all_items_partitions.partition, all_items_partitions.tstzrange, all_items_partitions.pstart, all_items_partitions.pend, all_items_partitions.est_cnt FROM pgstac.all_items_partitions WHERE (all_items_partitions.est_cnt > 0); CREATE OR REPLACE FUNCTION pgstac.items_path(dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text) RETURNS record LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE AS $function$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_ignore_insert_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_insert_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; p record; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_upsert_insert_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.partition_cursor(_where text DEFAULT 'TRUE'::text, _orderby text DEFAULT 'datetime DESC, id DESC'::text) RETURNS SETOF refcursor LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' AS $function$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partition_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SET jit TO 'off' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; curs = create_cursor(query); --OPEN curs LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context() != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.sort_sqlorderby(_search jsonb DEFAULT NULL::jsonb, reverse boolean DEFAULT false) RETURNS text LANGUAGE sql AS $function$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce(field_orderby((items_path(value->>'field')).path_txt), (items_path(value->>'field')).path) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $function$ ; DO $$ DECLARE partition text; topartition text; q text; matches text[]; BEGIN FOR partition IN SELECT (to_json(parse_ident(c.oid::pg_catalog.regclass::text)))->>-1 FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid=i.inhrelid and i.inhparent='items'::regclass LOOP RAISE NOTICE 'Partition: %', partition; topartition := (to_json(regexp_split_to_array(partition, '\.')))->>-1; IF partition != topartition THEN RAISE NOTICE 'Renaming partition % to %', partition, topartition; q := format($q$ ALTER TABLE %I RENAME TO %I; $q$, partition, topartition); RAISE NOTICE '%', q; EXECUTE q; END IF; END LOOP; END; $$ LANGUAGE PLPGSQL; SELECT partition_checks(partition) FROM all_items_partitions; SELECT set_version('0.3.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.4.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text, datetime timestamptz DEFAULT now() NOT NULL ); CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN IF _orderby ILIKE 'datetime d%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); ELSIF _orderby ILIKE 'datetime a%' THEN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange ASC ; $q$); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND %s ORDER BY %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where, _orderby ); RETURN NEXT query; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partion_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT DO NOTHING ; DELETE FROM items_staging_ignore; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; BEGIN SELECT min(stac_datetime(content)), max(stac_datetime(content)) INTO mindate, maxdate FROM newdata; PERFORM items_partition_create(mindate, maxdate); INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging_upsert; PERFORM analyze_empty_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; -- format($F$ %s = %%s $F$, field); RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "like": "%s LIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":"lower(%s)" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL 1: %', search; -- Convert any old style stac query to cql search := query_to_cqlfilter(search); RAISE NOTICE 'SEARCH CQL 2: %', search; -- Convert item,collection,datetime,bbox,intersects to cql search := add_filters_to_cql(search); RAISE NOTICE 'SEARCH CQL Final: %', search; _where := cql_query_op(search->'filter'); IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT (items_path(value->>'field')).path as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb) RETURNS text AS $$ SELECT md5(search_tohash($1)::text); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, total_count bigint ); CREATE OR REPLACE FUNCTION search_query(_search jsonb = '{}'::jsonb, updatestats boolean DEFAULT false) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; BEGIN INSERT INTO searches (search) VALUES (search_tohash(_search)) ON CONFLICT DO NOTHING RETURNING * INTO search; IF search.hash IS NULL THEN SELECT * INTO search FROM searches WHERE hash=search_hash(_search); END IF; IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; IF search.statslastupdated IS NULL OR age(search.statslastupdated) > '1 day'::interval OR (_search ? 'context' AND search.total_count IS NULL) THEN updatestats := TRUE; END IF; IF updatestats THEN -- Get Estimated Stats RAISE NOTICE 'Getting stats for %', search._where; search.estimated_count := estimated_count(search._where); RAISE NOTICE 'Estimated Count: %', search.estimated_count; IF _search ? 'context' OR search.estimated_count < 10000 THEN --search.total_count := partition_count(search._where); EXECUTE format( 'SELECT count(*) FROM items WHERE %s', search._where ) INTO search.total_count; RAISE NOTICE 'Actual Count: %', search.total_count; ELSE search.total_count := NULL; END IF; search.statslastupdated := now(); END IF; search.lastused := now(); search.usecount := coalesce(search.usecount,0) + 1; RAISE NOTICE 'SEARCH: %', search; UPDATE searches SET _where = search._where, orderby = search.orderby, lastused = search.lastused, usecount = search.usecount, statslastupdated = search.statslastupdated, estimated_count = search.estimated_count, total_count = search.total_count WHERE hash = search.hash ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; total_count := coalesce(searches.total_count, searches.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby) LOOP timer := clock_timestamp(); query := format('%s LIMIT %L', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; curs = create_cursor(query); LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; RAISE NOTICE 'Query took %', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb[] := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); IF fields IS NOT NULL THEN IF fields ? 'fields' THEN fields := fields->'fields'; END IF; IF fields ? 'exclude' THEN excludes=textarr(fields->'exclude'); END IF; IF fields ? 'include' THEN includes=textarr(fields->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; END IF; RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; curs = create_cursor(query); LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; IF fields IS NOT NULL THEN out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); ELSE out_records := out_records || iter_record.content; END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', array_to_json(out_records)::jsonb ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; INSERT INTO pgstac.migrations (version) VALUES ('0.3.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.5-0.3.6.sql ================================================ SET SEARCH_PATH to pgstac, public; CREATE INDEX search_wheres_partitions ON pgstac.search_wheres USING gin (partitions); set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.validate_constraints() RETURNS void LANGUAGE plpgsql AS $function$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I.%I VALIDATE CONSTRAINT %I;', current_database(), nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated IS FALSE AND nsp.nspname = 'pgstac' LOOP EXECUTE q; END LOOP; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.drop_partition_constraints(partition text) RETURNS void LANGUAGE plpgsql AS $function$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, DROP CONSTRAINT IF EXISTS %I; $q$, partition, end_datetime_constraint, collections_constraint ); EXECUTE q; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_ignore_insert_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE partitions && _partitions ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_insert_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE partitions && _partitions ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_upsert_insert_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE partitions && _partitions ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.partition_checks(partition text, OUT min_datetime timestamp with time zone, OUT max_datetime timestamp with time zone, OUT min_end_datetime timestamp with time zone, OUT max_end_datetime timestamp with time zone, OUT collections text[], OUT cnt bigint) RETURNS record LANGUAGE plpgsql AS $function$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; q := format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime), array_agg(DISTINCT collection_id), count(*) FROM %I; $q$, partition ); EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; RAISE NOTICE '% % % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt, ftime(); IF cnt IS NULL or cnt = 0 THEN RAISE NOTICE 'Partition % is empty, removing...', partition; q := format($q$ DROP TABLE IF EXISTS %I; $q$, partition ); EXECUTE q; RETURN; END IF; RAISE NOTICE 'Running Constraint DDL %', ftime(); q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((end_datetime >= %L) AND (end_datetime <= %L)) NOT VALID, DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((collection_id = ANY(%L))) NOT VALID; $q$, partition, end_datetime_constraint, end_datetime_constraint, min_end_datetime, max_end_datetime, collections_constraint, collections_constraint, collections, partition ); RAISE NOTICE 'q: %', q; EXECUTE q; RAISE NOTICE 'Returning %', ftime(); RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SET jit TO 'off' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; SELECT jsonb_agg(content) INTO out_records FROM results; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context() != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; SELECT set_version('0.3.6'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.5.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_setting(IN setting text, INOUT _default anynonarray = null::text ) AS $$ DECLARE _type text; BEGIN SELECT pg_typeof(_default) INTO _type; IF _type = 'unknown' THEN _type='text'; END IF; EXECUTE format($q$ SELECT COALESCE( CAST(current_setting($1,TRUE) AS %s), $2 ) $q$, _type) INTO _default USING setting, _default ; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION context() RETURNS text AS $$ SELECT get_setting('pgstac.context','off'::text); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count() RETURNS int AS $$ SELECT get_setting('pgstac.context_estimated_count', 100000::int); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_cost() RETURNS float AS $$ SELECT get_setting('pgstac.context_estimated_cost', 1000000::float); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_stats_ttl() RETURNS interval AS $$ SELECT get_setting('pgstac.context_stats_ttl', '1 day'::interval); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE 'sql' STRICT IMMUTABLE; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_queries; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT '{items}' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p text; cursors refcursor; dstart timestamptz; dend timestamptz; step interval := '10 weeks'::interval; BEGIN IF _orderby ILIKE 'datetime d%' THEN partitions := partitions; ELSIF _orderby ILIKE 'datetime a%' THEN partitions := array_reverse(partitions); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RAISE NOTICE 'PARTITIONS ---> %',partitions; IF cardinality(partitions) > 0 THEN FOREACH p IN ARRAY partitions --EXECUTE partition_query LOOP query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s $q$, p, _where, _orderby ); RETURN NEXT query; END LOOP; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partition_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION drop_partition_constraints(IN partition text) RETURNS VOID AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, partition, end_datetime_constraint, partition, collections_constraint ); EXECUTE q; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_checks; CREATE OR REPLACE FUNCTION partition_checks( IN partition text, OUT min_datetime timestamptz, OUT max_datetime timestamptz, OUT min_end_datetime timestamptz, OUT max_end_datetime timestamptz, OUT collections text[], OUT cnt bigint ) RETURNS RECORD AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; q := format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime), array_agg(DISTINCT collection_id), count(*) FROM %I; $q$, partition ); EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; RAISE NOTICE '% % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; IF cnt IS NULL or cnt = 0 THEN RAISE NOTICE 'Partition % is empty, removing...', partition; q := format($q$ DROP TABLE IF EXISTS %I; $q$, partition ); EXECUTE q; RETURN; END IF; q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I check((end_datetime >= %L) AND (end_datetime <= %L)); ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I check((collection_id = ANY(%L))); ANALYZE %I; $q$, partition, end_datetime_constraint, partition, end_datetime_constraint, min_end_datetime, max_end_datetime, partition, collections_constraint, partition, collections_constraint, collections, partition ); EXECUTE q; RETURN; END; $$ LANGUAGE PLPGSQL; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE mindate timestamptz; maxdate timestamptz; partition text; p record; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), t[1]::timestamptz as pstart, t[2]::timestamptz as pend, est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "like": "%s LIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":"lower(%s)" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL 1: %', search; -- Convert any old style stac query to cql search := query_to_cqlfilter(search); RAISE NOTICE 'SEARCH CQL 2: %', search; -- Convert item,collection,datetime,bbox,intersects to cql search := add_filters_to_cql(search); RAISE NOTICE 'SEARCH CQL Final: %', search; _where := cql_query_op(search->'filter'); IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION field_orderby(p text) RETURNS text AS $$ WITH t AS ( SELECT replace(trim(substring(indexdef from 'btree \((.*)\)')),' ','')as s FROM pg_indexes WHERE schemaname='pgstac' AND tablename='items' AND indexdef ~* 'btree' AND indexdef ~* 'properties' ) SELECT s FROM t WHERE strpos(s, lower(trim(p)))>0; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce(field_orderby((items_path(value->>'field')).path_txt), (items_path(value->>'field')).path) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( _where text PRIMARY KEY, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl() OR (context() != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context() = 'on' OR ( context() = 'auto' AND ( sw.estimated_count < context_estimated_count() OR sw.estimated_cost < context_estimated_cost() ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres SELECT sw.* ON CONFLICT (_where) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; CREATE OR REPLACE FUNCTION items_count(_where text) RETURNS bigint AS $$ DECLARE cnt bigint; BEGIN EXECUTE format('SELECT count(*) FROM items WHERE %s', _where) INTO cnt; RETURN cnt; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; curs = create_cursor(query); --OPEN curs LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context() != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off ; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb[] := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); IF fields IS NOT NULL THEN IF fields ? 'fields' THEN fields := fields->'fields'; END IF; IF fields ? 'exclude' THEN excludes=textarr(fields->'exclude'); END IF; IF fields ? 'include' THEN includes=textarr(fields->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; END IF; RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; curs = create_cursor(query); LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; IF fields IS NOT NULL THEN out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); ELSE out_records := out_records || iter_record.content; END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', array_to_json(out_records)::jsonb ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; SELECT set_version('0.3.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.6-0.4.0.sql ================================================ SET SEARCH_PATH to pgstac, public; drop function if exists "pgstac"."context"(); drop function if exists "pgstac"."context_estimated_cost"(); drop function if exists "pgstac"."context_estimated_count"(); drop function if exists "pgstac"."context_stats_ttl"(); drop function if exists "pgstac"."get_setting"(setting text, INOUT _default anynonarray); drop function if exists "pgstac"."where_stats"(inwhere text, updatestats boolean); create table "pgstac"."pgstac_settings" ( "name" text not null, "value" text not null ); CREATE UNIQUE INDEX pgstac_settings_pkey ON pgstac.pgstac_settings USING btree (name); alter table "pgstac"."pgstac_settings" add constraint "pgstac_settings_pkey" PRIMARY KEY using index "pgstac_settings_pkey"; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.context(conf jsonb DEFAULT NULL::jsonb) RETURNS text LANGUAGE sql AS $function$ SELECT get_setting('context', conf); $function$ ; CREATE OR REPLACE FUNCTION pgstac.context_estimated_cost(conf jsonb DEFAULT NULL::jsonb) RETURNS double precision LANGUAGE sql AS $function$ SELECT get_setting('context_estimated_cost', conf)::float; $function$ ; CREATE OR REPLACE FUNCTION pgstac.context_estimated_count(conf jsonb DEFAULT NULL::jsonb) RETURNS integer LANGUAGE sql AS $function$ SELECT get_setting('context_estimated_count', conf)::int; $function$ ; CREATE OR REPLACE FUNCTION pgstac.context_stats_ttl(conf jsonb DEFAULT NULL::jsonb) RETURNS interval LANGUAGE sql AS $function$ SELECT get_setting('context_stats_ttl', conf)::interval; $function$ ; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, recursion integer DEFAULT 0) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE args jsonb := j->'args'; jtype text := jsonb_typeof(j->'args'); op text := lower(j->>'op'); arg jsonb; argtext text; argstext text[] := '{}'::text[]; inobj jsonb; _numeric text := ''; ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN (%2$s)[1] AND (%2$s)[2]", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; BEGIN RAISE NOTICE 'j: %s', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RAISE NOTICE 'upper %s',jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ; RETURN cql2_query( jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ); END IF; IF j ? 'lower' THEN RETURN cql2_query( jsonb_build_object( 'op', 'lower', 'args', jsonb_build_array( j-> 'lower') ) ); END IF; IF j ? 'args' AND jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; -- END Cases where no further nesting is expected IF j ? 'op' THEN -- Special case to use JSONB index for equality IF op = 'eq' AND args->0 ? 'property' AND jsonb_typeof(args->1) IN ('number', 'string') AND (items_path(args->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(args->0->>'property')).eq, args->1); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; -- In Query - separate into separate eq statements so that we can use eq jsonb optimization IF op = 'in' THEN RAISE NOTICE '% IN args: %', repeat(' ', recursion), args; SELECT INTO inobj jsonb_agg( jsonb_build_object( 'op', 'eq', 'args', jsonb_build_array( args->0 , v) ) ) FROM jsonb_array_elements( args->1) v; RETURN cql2_query(jsonb_build_object('op','or','args',inobj)); END IF; END IF; IF j ? 'property' THEN RETURN (items_path(j->>'property')).path_txt; END IF; RAISE NOTICE '%jtype: %',repeat(' ', recursion), jtype; IF jsonb_typeof(j) = 'number' THEN RETURN format('%L::numeric', j->>0); END IF; IF jsonb_typeof(j) = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jsonb_typeof(j) = 'array' THEN IF j @? '$[*] ? (@.type() == "number")' THEN RETURN CONCAT(quote_literal(textarr(j)::text), '::numeric[]'); ELSE RETURN CONCAT(quote_literal(textarr(j)::text), '::text[]'); END IF; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; RAISE NOTICE '%beforeargs op: %, args: %',repeat(' ', recursion), op, args; IF j ? 'args' THEN FOR arg in SELECT * FROM jsonb_array_elements(args) LOOP argtext := cql2_query(arg, recursion + 1); RAISE NOTICE '% -- arg: %, argtext: %', repeat(' ', recursion), arg, argtext; argstext := argstext || argtext; END LOOP; END IF; RAISE NOTICE '%afterargs op: %, argstext: %',repeat(' ', recursion), op, argstext; IF op IN ('and', 'or') THEN RAISE NOTICE 'inand op: %, argstext: %', op, argstext; SELECT concat(' ( ',array_to_string(array_agg(e), concat(' ',op,' ')),' ) ') INTO ret FROM unnest(argstext) e; RETURN ret; END IF; IF ops ? op THEN IF argstext[2] ~* 'numeric' THEN argstext := ARRAY[concat('(',argstext[1],')::numeric')] || argstext[2:3]; END IF; RETURN format(concat('(',ops->>op,')'), VARIADIC argstext); END IF; RAISE NOTICE '%op: %, argstext: %',repeat(' ', recursion), op, argstext; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_setting(_setting text, conf jsonb DEFAULT NULL::jsonb) RETURNS text LANGUAGE sql AS $function$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac_settings WHERE name=_setting) ); $function$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) RETURNS search_wheres LANGUAGE plpgsql AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(conf), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(conf), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl(conf) OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context(conf) = 'on' OR ( context(conf) = 'auto' AND ( sw.estimated_count < context_estimated_count(conf) OR sw.estimated_cost < context_estimated_cost(conf) ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres SELECT sw.* ON CONFLICT (_where) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.cql_query_op(j jsonb, _op text DEFAULT NULL::text) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- for in, convert value, list to array syntax to match other ops IF op = 'in' and j ? 'value' and j ? 'list' THEN j := jsonb_build_array( j->'value', j->'list'); jtype := 'array'; RAISE NOTICE 'IN: j: %, jtype: %', j, jtype; END IF; IF op = 'between' and j ? 'value' and j ? 'lower' and j ? 'upper' THEN j := jsonb_build_array( j->'value', j->'lower', j->'upper'); jtype := 'array'; RAISE NOTICE 'BETWEEN: j: %, jtype: %', j, jtype; END IF; IF op = 'not' AND jtype = 'object' THEN j := jsonb_build_array( j ); jtype := 'array'; RAISE NOTICE 'NOT: j: %, jtype: %', j, jtype; END IF; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.cql_to_where(_search jsonb DEFAULT '{}'::jsonb) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL Final: %', search; IF (search ? 'filter-lang' AND search->>'filter-lang' = 'cql-json') OR get_setting('default-filter-lang', _search->'conf')='cql-json' THEN search := query_to_cqlfilter(search); search := add_filters_to_cql(search); _where := cql_query_op(search->'filter'); ELSE _where := cql2_query(search->'filter'); END IF; IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_ignore_insert_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_insert_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_upsert_insert_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SET jit TO 'off' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; SELECT jsonb_agg(content) INTO out_records FROM results; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) RETURNS searches LANGUAGE plpgsql AS $function$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $function$ ; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '1000000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json') ON CONFLICT DO NOTHING ; SELECT set_version('0.4.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.3.6.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_setting(IN setting text, INOUT _default anynonarray = null::text ) AS $$ DECLARE _type text; BEGIN SELECT pg_typeof(_default) INTO _type; IF _type = 'unknown' THEN _type='text'; END IF; EXECUTE format($q$ SELECT COALESCE( CAST(current_setting($1,TRUE) AS %s), $2 ) $q$, _type) INTO _default USING setting, _default ; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION context() RETURNS text AS $$ SELECT get_setting('pgstac.context','off'::text); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count() RETURNS int AS $$ SELECT get_setting('pgstac.context_estimated_count', 100000::int); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_cost() RETURNS float AS $$ SELECT get_setting('pgstac.context_estimated_cost', 1000000::float); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_stats_ttl() RETURNS interval AS $$ SELECT get_setting('pgstac.context_stats_ttl', '1 day'::interval); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE 'sql' STRICT IMMUTABLE; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_queries; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT '{items}' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p text; cursors refcursor; dstart timestamptz; dend timestamptz; step interval := '10 weeks'::interval; BEGIN IF _orderby ILIKE 'datetime d%' THEN partitions := partitions; ELSIF _orderby ILIKE 'datetime a%' THEN partitions := array_reverse(partitions); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RAISE NOTICE 'PARTITIONS ---> %',partitions; IF cardinality(partitions) > 0 THEN FOREACH p IN ARRAY partitions --EXECUTE partition_query LOOP query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s $q$, p, _where, _orderby ); RETURN NEXT query; END LOOP; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partition_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION drop_partition_constraints(IN partition text) RETURNS VOID AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, DROP CONSTRAINT IF EXISTS %I; $q$, partition, end_datetime_constraint, collections_constraint ); EXECUTE q; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_checks; CREATE OR REPLACE FUNCTION partition_checks( IN partition text, OUT min_datetime timestamptz, OUT max_datetime timestamptz, OUT min_end_datetime timestamptz, OUT max_end_datetime timestamptz, OUT collections text[], OUT cnt bigint ) RETURNS RECORD AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; q := format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime), array_agg(DISTINCT collection_id), count(*) FROM %I; $q$, partition ); EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; RAISE NOTICE '% % % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt, ftime(); IF cnt IS NULL or cnt = 0 THEN RAISE NOTICE 'Partition % is empty, removing...', partition; q := format($q$ DROP TABLE IF EXISTS %I; $q$, partition ); EXECUTE q; RETURN; END IF; RAISE NOTICE 'Running Constraint DDL %', ftime(); q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((end_datetime >= %L) AND (end_datetime <= %L)) NOT VALID, DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((collection_id = ANY(%L))) NOT VALID; $q$, partition, end_datetime_constraint, end_datetime_constraint, min_end_datetime, max_end_datetime, collections_constraint, collections_constraint, collections, partition ); RAISE NOTICE 'q: %', q; EXECUTE q; RAISE NOTICE 'Returning %', ftime(); RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION validate_constraints() RETURNS VOID AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I.%I VALIDATE CONSTRAINT %I;', current_database(), nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated IS FALSE AND nsp.nspname = 'pgstac' LOOP EXECUTE q; END LOOP; END; $$ LANGUAGE PLPGSQL; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE partitions && _partitions ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE partitions && _partitions ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE partitions && _partitions ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), t[1]::timestamptz as pstart, t[2]::timestamptz as pend, est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "like": "%s LIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":"lower(%s)" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL 1: %', search; -- Convert any old style stac query to cql search := query_to_cqlfilter(search); RAISE NOTICE 'SEARCH CQL 2: %', search; -- Convert item,collection,datetime,bbox,intersects to cql search := add_filters_to_cql(search); RAISE NOTICE 'SEARCH CQL Final: %', search; _where := cql_query_op(search->'filter'); IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION field_orderby(p text) RETURNS text AS $$ WITH t AS ( SELECT replace(trim(substring(indexdef from 'btree \((.*)\)')),' ','')as s FROM pg_indexes WHERE schemaname='pgstac' AND tablename='items' AND indexdef ~* 'btree' AND indexdef ~* 'properties' ) SELECT s FROM t WHERE strpos(s, lower(trim(p)))>0; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce(field_orderby((items_path(value->>'field')).path_txt), (items_path(value->>'field')).path) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( _where text PRIMARY KEY, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl() OR (context() != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context() = 'on' OR ( context() = 'auto' AND ( sw.estimated_count < context_estimated_count() OR sw.estimated_cost < context_estimated_cost() ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres SELECT sw.* ON CONFLICT (_where) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; CREATE OR REPLACE FUNCTION items_count(_where text) RETURNS bigint AS $$ DECLARE cnt bigint; BEGIN EXECUTE format('SELECT count(*) FROM items WHERE %s', _where) INTO cnt; RETURN cnt; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; SELECT jsonb_agg(content) INTO out_records FROM results; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context() != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off ; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb[] := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); IF fields IS NOT NULL THEN IF fields ? 'fields' THEN fields := fields->'fields'; END IF; IF fields ? 'exclude' THEN excludes=textarr(fields->'exclude'); END IF; IF fields ? 'include' THEN includes=textarr(fields->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; END IF; RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; curs = create_cursor(query); LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; IF fields IS NOT NULL THEN out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); ELSE out_records := out_records || iter_record.content; END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', array_to_json(out_records)::jsonb ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; SELECT set_version('0.3.6'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.0-0.4.1.sql ================================================ SET SEARCH_PATH to pgstac, public; alter table "pgstac"."search_wheres" drop constraint "search_wheres_pkey"; drop index if exists "pgstac"."search_wheres_pkey"; alter table "pgstac"."search_wheres" add column "id" bigint generated always as identity not null; CREATE UNIQUE INDEX search_wheres_where ON pgstac.search_wheres USING btree (md5(_where)); CREATE UNIQUE INDEX search_wheres_pkey ON pgstac.search_wheres USING btree (id); alter table "pgstac"."search_wheres" add constraint "search_wheres_pkey" PRIMARY KEY using index "search_wheres_pkey"; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.cql_to_where(_search jsonb DEFAULT '{}'::jsonb) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE filterlang text; search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL Final: %', search; filterlang := COALESCE( search->>'filter-lang', get_setting('default-filter-lang', _search->'conf') ); IF filterlang = 'cql-json' THEN search := query_to_cqlfilter(search); search := add_filters_to_cql(search); _where := cql_query_op(search->'filter'); ELSE _where := cql2_query(search->'filter'); END IF; IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) RETURNS search_wheres LANGUAGE plpgsql AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(conf), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(conf), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl(conf) OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context(conf) = 'on' OR ( context(conf) = 'auto' AND ( sw.estimated_count < context_estimated_count(conf) OR sw.estimated_cost < context_estimated_cost(conf) ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $function$ ; SELECT set_version('0.4.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.0.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '1000000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context(); CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT get_setting('context', conf); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_count(); CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE 'sql' STRICT IMMUTABLE; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_queries; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT '{items}' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p text; cursors refcursor; dstart timestamptz; dend timestamptz; step interval := '10 weeks'::interval; BEGIN IF _orderby ILIKE 'datetime d%' THEN partitions := partitions; ELSIF _orderby ILIKE 'datetime a%' THEN partitions := array_reverse(partitions); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RAISE NOTICE 'PARTITIONS ---> %',partitions; IF cardinality(partitions) > 0 THEN FOREACH p IN ARRAY partitions --EXECUTE partition_query LOOP query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s $q$, p, _where, _orderby ); RETURN NEXT query; END LOOP; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partition_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION drop_partition_constraints(IN partition text) RETURNS VOID AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, DROP CONSTRAINT IF EXISTS %I; $q$, partition, end_datetime_constraint, collections_constraint ); EXECUTE q; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_checks; CREATE OR REPLACE FUNCTION partition_checks( IN partition text, OUT min_datetime timestamptz, OUT max_datetime timestamptz, OUT min_end_datetime timestamptz, OUT max_end_datetime timestamptz, OUT collections text[], OUT cnt bigint ) RETURNS RECORD AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; q := format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime), array_agg(DISTINCT collection_id), count(*) FROM %I; $q$, partition ); EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; RAISE NOTICE '% % % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt, ftime(); IF cnt IS NULL or cnt = 0 THEN RAISE NOTICE 'Partition % is empty, removing...', partition; q := format($q$ DROP TABLE IF EXISTS %I; $q$, partition ); EXECUTE q; RETURN; END IF; RAISE NOTICE 'Running Constraint DDL %', ftime(); q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((end_datetime >= %L) AND (end_datetime <= %L)) NOT VALID, DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((collection_id = ANY(%L))) NOT VALID; $q$, partition, end_datetime_constraint, end_datetime_constraint, min_end_datetime, max_end_datetime, collections_constraint, collections_constraint, collections, partition ); RAISE NOTICE 'q: %', q; EXECUTE q; RAISE NOTICE 'Returning %', ftime(); RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION validate_constraints() RETURNS VOID AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I.%I VALIDATE CONSTRAINT %I;', current_database(), nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated IS FALSE AND nsp.nspname = 'pgstac' LOOP EXECUTE q; END LOOP; END; $$ LANGUAGE PLPGSQL; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), t[1]::timestamptz as pstart, t[2]::timestamptz as pend, est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- for in, convert value, list to array syntax to match other ops IF op = 'in' and j ? 'value' and j ? 'list' THEN j := jsonb_build_array( j->'value', j->'list'); jtype := 'array'; RAISE NOTICE 'IN: j: %, jtype: %', j, jtype; END IF; IF op = 'between' and j ? 'value' and j ? 'lower' and j ? 'upper' THEN j := jsonb_build_array( j->'value', j->'lower', j->'upper'); jtype := 'array'; RAISE NOTICE 'BETWEEN: j: %, jtype: %', j, jtype; END IF; IF op = 'not' AND jtype = 'object' THEN j := jsonb_build_array( j ); jtype := 'array'; RAISE NOTICE 'NOT: j: %, jtype: %', j, jtype; END IF; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ DROP FUNCTION IF EXISTS cql2_query; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, recursion int DEFAULT 0) RETURNS text AS $$ DECLARE args jsonb := j->'args'; jtype text := jsonb_typeof(j->'args'); op text := lower(j->>'op'); arg jsonb; argtext text; argstext text[] := '{}'::text[]; inobj jsonb; _numeric text := ''; ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN (%2$s)[1] AND (%2$s)[2]", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; BEGIN RAISE NOTICE 'j: %s', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RAISE NOTICE 'upper %s',jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ; RETURN cql2_query( jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ); END IF; IF j ? 'lower' THEN RETURN cql2_query( jsonb_build_object( 'op', 'lower', 'args', jsonb_build_array( j-> 'lower') ) ); END IF; IF j ? 'args' AND jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; -- END Cases where no further nesting is expected IF j ? 'op' THEN -- Special case to use JSONB index for equality IF op = 'eq' AND args->0 ? 'property' AND jsonb_typeof(args->1) IN ('number', 'string') AND (items_path(args->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(args->0->>'property')).eq, args->1); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; -- In Query - separate into separate eq statements so that we can use eq jsonb optimization IF op = 'in' THEN RAISE NOTICE '% IN args: %', repeat(' ', recursion), args; SELECT INTO inobj jsonb_agg( jsonb_build_object( 'op', 'eq', 'args', jsonb_build_array( args->0 , v) ) ) FROM jsonb_array_elements( args->1) v; RETURN cql2_query(jsonb_build_object('op','or','args',inobj)); END IF; END IF; IF j ? 'property' THEN RETURN (items_path(j->>'property')).path_txt; END IF; RAISE NOTICE '%jtype: %',repeat(' ', recursion), jtype; IF jsonb_typeof(j) = 'number' THEN RETURN format('%L::numeric', j->>0); END IF; IF jsonb_typeof(j) = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jsonb_typeof(j) = 'array' THEN IF j @? '$[*] ? (@.type() == "number")' THEN RETURN CONCAT(quote_literal(textarr(j)::text), '::numeric[]'); ELSE RETURN CONCAT(quote_literal(textarr(j)::text), '::text[]'); END IF; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; RAISE NOTICE '%beforeargs op: %, args: %',repeat(' ', recursion), op, args; IF j ? 'args' THEN FOR arg in SELECT * FROM jsonb_array_elements(args) LOOP argtext := cql2_query(arg, recursion + 1); RAISE NOTICE '% -- arg: %, argtext: %', repeat(' ', recursion), arg, argtext; argstext := argstext || argtext; END LOOP; END IF; RAISE NOTICE '%afterargs op: %, argstext: %',repeat(' ', recursion), op, argstext; IF op IN ('and', 'or') THEN RAISE NOTICE 'inand op: %, argstext: %', op, argstext; SELECT concat(' ( ',array_to_string(array_agg(e), concat(' ',op,' ')),' ) ') INTO ret FROM unnest(argstext) e; RETURN ret; END IF; IF ops ? op THEN IF argstext[2] ~* 'numeric' THEN argstext := ARRAY[concat('(',argstext[1],')::numeric')] || argstext[2:3]; END IF; RETURN format(concat('(',ops->>op,')'), VARIADIC argstext); END IF; RAISE NOTICE '%op: %, argstext: %',repeat(' ', recursion), op, argstext; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL Final: %', search; IF (search ? 'filter-lang' AND search->>'filter-lang' = 'cql-json') OR get_setting('default-filter-lang', _search->'conf')='cql-json' THEN search := query_to_cqlfilter(search); search := add_filters_to_cql(search); _where := cql_query_op(search->'filter'); ELSE _where := cql2_query(search->'filter'); END IF; IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION field_orderby(p text) RETURNS text AS $$ WITH t AS ( SELECT replace(trim(substring(indexdef from 'btree \((.*)\)')),' ','')as s FROM pg_indexes WHERE schemaname='pgstac' AND tablename='items' AND indexdef ~* 'btree' AND indexdef ~* 'properties' ) SELECT s FROM t WHERE strpos(s, lower(trim(p)))>0; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce(field_orderby((items_path(value->>'field')).path_txt), (items_path(value->>'field')).path) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( _where text PRIMARY KEY, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(conf), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(conf), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl(conf) OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context(conf) = 'on' OR ( context(conf) = 'auto' AND ( sw.estimated_count < context_estimated_count(conf) OR sw.estimated_cost < context_estimated_cost(conf) ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres SELECT sw.* ON CONFLICT (_where) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; CREATE OR REPLACE FUNCTION items_count(_where text) RETURNS bigint AS $$ DECLARE cnt bigint; BEGIN EXECUTE format('SELECT count(*) FROM items WHERE %s', _where) INTO cnt; RETURN cnt; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; SELECT jsonb_agg(content) INTO out_records FROM results; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off ; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb[] := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); IF fields IS NOT NULL THEN IF fields ? 'fields' THEN fields := fields->'fields'; END IF; IF fields ? 'exclude' THEN excludes=textarr(fields->'exclude'); END IF; IF fields ? 'include' THEN includes=textarr(fields->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; END IF; RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; curs = create_cursor(query); LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; IF fields IS NOT NULL THEN out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); ELSE out_records := out_records || iter_record.content; END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', array_to_json(out_records)::jsonb ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; SELECT set_version('0.4.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.1-0.4.2.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, recursion integer DEFAULT 0) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE args jsonb := j->'args'; jtype text := jsonb_typeof(j->'args'); op text := lower(j->>'op'); arg jsonb; argtext text; argstext text[] := '{}'::text[]; inobj jsonb; _numeric text := ''; ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN (%2$s)[1] AND (%2$s)[2]", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; BEGIN RAISE NOTICE 'j: %s', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RAISE NOTICE 'upper %s',jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ; RETURN cql2_query( jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ); END IF; IF j ? 'lower' THEN RETURN cql2_query( jsonb_build_object( 'op', 'lower', 'args', jsonb_build_array( j-> 'lower') ) ); END IF; IF j ? 'args' AND jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; -- END Cases where no further nesting is expected IF j ? 'op' THEN -- Special case to use JSONB index for equality IF op = 'eq' AND args->0 ? 'property' AND jsonb_typeof(args->1) IN ('number', 'string') AND (items_path(args->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(args->0->>'property')).eq, args->1); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; -- In Query - separate into separate eq statements so that we can use eq jsonb optimization IF op = 'in' THEN RAISE NOTICE '% IN args: %', repeat(' ', recursion), args; SELECT INTO inobj jsonb_agg( jsonb_build_object( 'op', 'eq', 'args', jsonb_build_array( args->0 , v) ) ) FROM jsonb_array_elements( args->1) v; RETURN cql2_query(jsonb_build_object('op','or','args',inobj)); END IF; END IF; IF j ? 'property' THEN RETURN (items_path(j->>'property')).path_txt; END IF; IF j ? 'timestamp' THEN RETURN quote_literal(j->>'timestamp'); END IF; RAISE NOTICE '%jtype: %',repeat(' ', recursion), jtype; IF jsonb_typeof(j) = 'number' THEN RETURN format('%L::numeric', j->>0); END IF; IF jsonb_typeof(j) = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jsonb_typeof(j) = 'array' THEN IF j @? '$[*] ? (@.type() == "number")' THEN RETURN CONCAT(quote_literal(textarr(j)::text), '::numeric[]'); ELSE RETURN CONCAT(quote_literal(textarr(j)::text), '::text[]'); END IF; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; RAISE NOTICE '%beforeargs op: %, args: %',repeat(' ', recursion), op, args; IF j ? 'args' THEN FOR arg in SELECT * FROM jsonb_array_elements(args) LOOP argtext := cql2_query(arg, recursion + 1); RAISE NOTICE '% -- arg: %, argtext: %', repeat(' ', recursion), arg, argtext; argstext := argstext || argtext; END LOOP; END IF; RAISE NOTICE '%afterargs op: %, argstext: %',repeat(' ', recursion), op, argstext; IF op IN ('and', 'or') THEN RAISE NOTICE 'inand op: %, argstext: %', op, argstext; SELECT concat(' ( ',array_to_string(array_agg(e), concat(' ',op,' ')),' ) ') INTO ret FROM unnest(argstext) e; RETURN ret; END IF; IF ops ? op THEN IF argstext[2] ~* 'numeric' THEN argstext := ARRAY[concat('(',argstext[1],')::numeric')] || argstext[2:3]; END IF; RETURN format(concat('(',ops->>op,')'), VARIADIC argstext); END IF; RAISE NOTICE '%op: %, argstext: %',repeat(' ', recursion), op, argstext; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.parse_dtrange(_indate jsonb, OUT _tstzrange tstzrange) RETURNS tstzrange LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ WITH t AS ( SELECT CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp', 'infinity'] WHEN _indate ? 'interval' THEN textarr(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $function$ ; SELECT set_version('0.4.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.1.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '1000000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context(); CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT get_setting('context', conf); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_count(); CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE 'sql' STRICT IMMUTABLE; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_queries; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT '{items}' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p text; cursors refcursor; dstart timestamptz; dend timestamptz; step interval := '10 weeks'::interval; BEGIN IF _orderby ILIKE 'datetime d%' THEN partitions := partitions; ELSIF _orderby ILIKE 'datetime a%' THEN partitions := array_reverse(partitions); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RAISE NOTICE 'PARTITIONS ---> %',partitions; IF cardinality(partitions) > 0 THEN FOREACH p IN ARRAY partitions --EXECUTE partition_query LOOP query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s $q$, p, _where, _orderby ); RETURN NEXT query; END LOOP; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partition_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION drop_partition_constraints(IN partition text) RETURNS VOID AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, DROP CONSTRAINT IF EXISTS %I; $q$, partition, end_datetime_constraint, collections_constraint ); EXECUTE q; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_checks; CREATE OR REPLACE FUNCTION partition_checks( IN partition text, OUT min_datetime timestamptz, OUT max_datetime timestamptz, OUT min_end_datetime timestamptz, OUT max_end_datetime timestamptz, OUT collections text[], OUT cnt bigint ) RETURNS RECORD AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; q := format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime), array_agg(DISTINCT collection_id), count(*) FROM %I; $q$, partition ); EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; RAISE NOTICE '% % % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt, ftime(); IF cnt IS NULL or cnt = 0 THEN RAISE NOTICE 'Partition % is empty, removing...', partition; q := format($q$ DROP TABLE IF EXISTS %I; $q$, partition ); EXECUTE q; RETURN; END IF; RAISE NOTICE 'Running Constraint DDL %', ftime(); q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((end_datetime >= %L) AND (end_datetime <= %L)) NOT VALID, DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((collection_id = ANY(%L))) NOT VALID; $q$, partition, end_datetime_constraint, end_datetime_constraint, min_end_datetime, max_end_datetime, collections_constraint, collections_constraint, collections, partition ); RAISE NOTICE 'q: %', q; EXECUTE q; RAISE NOTICE 'Returning %', ftime(); RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION validate_constraints() RETURNS VOID AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I.%I VALIDATE CONSTRAINT %I;', current_database(), nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated IS FALSE AND nsp.nspname = 'pgstac' LOOP EXECUTE q; END LOOP; END; $$ LANGUAGE PLPGSQL; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), t[1]::timestamptz as pstart, t[2]::timestamptz as pend, est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- for in, convert value, list to array syntax to match other ops IF op = 'in' and j ? 'value' and j ? 'list' THEN j := jsonb_build_array( j->'value', j->'list'); jtype := 'array'; RAISE NOTICE 'IN: j: %, jtype: %', j, jtype; END IF; IF op = 'between' and j ? 'value' and j ? 'lower' and j ? 'upper' THEN j := jsonb_build_array( j->'value', j->'lower', j->'upper'); jtype := 'array'; RAISE NOTICE 'BETWEEN: j: %, jtype: %', j, jtype; END IF; IF op = 'not' AND jtype = 'object' THEN j := jsonb_build_array( j ); jtype := 'array'; RAISE NOTICE 'NOT: j: %, jtype: %', j, jtype; END IF; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ DROP FUNCTION IF EXISTS cql2_query; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, recursion int DEFAULT 0) RETURNS text AS $$ DECLARE args jsonb := j->'args'; jtype text := jsonb_typeof(j->'args'); op text := lower(j->>'op'); arg jsonb; argtext text; argstext text[] := '{}'::text[]; inobj jsonb; _numeric text := ''; ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN (%2$s)[1] AND (%2$s)[2]", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; BEGIN RAISE NOTICE 'j: %s', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RAISE NOTICE 'upper %s',jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ; RETURN cql2_query( jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ); END IF; IF j ? 'lower' THEN RETURN cql2_query( jsonb_build_object( 'op', 'lower', 'args', jsonb_build_array( j-> 'lower') ) ); END IF; IF j ? 'args' AND jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; -- END Cases where no further nesting is expected IF j ? 'op' THEN -- Special case to use JSONB index for equality IF op = 'eq' AND args->0 ? 'property' AND jsonb_typeof(args->1) IN ('number', 'string') AND (items_path(args->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(args->0->>'property')).eq, args->1); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; -- In Query - separate into separate eq statements so that we can use eq jsonb optimization IF op = 'in' THEN RAISE NOTICE '% IN args: %', repeat(' ', recursion), args; SELECT INTO inobj jsonb_agg( jsonb_build_object( 'op', 'eq', 'args', jsonb_build_array( args->0 , v) ) ) FROM jsonb_array_elements( args->1) v; RETURN cql2_query(jsonb_build_object('op','or','args',inobj)); END IF; END IF; IF j ? 'property' THEN RETURN (items_path(j->>'property')).path_txt; END IF; RAISE NOTICE '%jtype: %',repeat(' ', recursion), jtype; IF jsonb_typeof(j) = 'number' THEN RETURN format('%L::numeric', j->>0); END IF; IF jsonb_typeof(j) = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jsonb_typeof(j) = 'array' THEN IF j @? '$[*] ? (@.type() == "number")' THEN RETURN CONCAT(quote_literal(textarr(j)::text), '::numeric[]'); ELSE RETURN CONCAT(quote_literal(textarr(j)::text), '::text[]'); END IF; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; RAISE NOTICE '%beforeargs op: %, args: %',repeat(' ', recursion), op, args; IF j ? 'args' THEN FOR arg in SELECT * FROM jsonb_array_elements(args) LOOP argtext := cql2_query(arg, recursion + 1); RAISE NOTICE '% -- arg: %, argtext: %', repeat(' ', recursion), arg, argtext; argstext := argstext || argtext; END LOOP; END IF; RAISE NOTICE '%afterargs op: %, argstext: %',repeat(' ', recursion), op, argstext; IF op IN ('and', 'or') THEN RAISE NOTICE 'inand op: %, argstext: %', op, argstext; SELECT concat(' ( ',array_to_string(array_agg(e), concat(' ',op,' ')),' ) ') INTO ret FROM unnest(argstext) e; RETURN ret; END IF; IF ops ? op THEN IF argstext[2] ~* 'numeric' THEN argstext := ARRAY[concat('(',argstext[1],')::numeric')] || argstext[2:3]; END IF; RETURN format(concat('(',ops->>op,')'), VARIADIC argstext); END IF; RAISE NOTICE '%op: %, argstext: %',repeat(' ', recursion), op, argstext; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE filterlang text; search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL Final: %', search; filterlang := COALESCE( search->>'filter-lang', get_setting('default-filter-lang', _search->'conf') ); IF filterlang = 'cql-json' THEN search := query_to_cqlfilter(search); search := add_filters_to_cql(search); _where := cql_query_op(search->'filter'); ELSE _where := cql2_query(search->'filter'); END IF; IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION field_orderby(p text) RETURNS text AS $$ WITH t AS ( SELECT replace(trim(substring(indexdef from 'btree \((.*)\)')),' ','')as s FROM pg_indexes WHERE schemaname='pgstac' AND tablename='items' AND indexdef ~* 'btree' AND indexdef ~* 'properties' ) SELECT s FROM t WHERE strpos(s, lower(trim(p)))>0; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce(field_orderby((items_path(value->>'field')).path_txt), (items_path(value->>'field')).path) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(conf), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(conf), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl(conf) OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context(conf) = 'on' OR ( context(conf) = 'auto' AND ( sw.estimated_count < context_estimated_count(conf) OR sw.estimated_cost < context_estimated_cost(conf) ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; CREATE OR REPLACE FUNCTION items_count(_where text) RETURNS bigint AS $$ DECLARE cnt bigint; BEGIN EXECUTE format('SELECT count(*) FROM items WHERE %s', _where) INTO cnt; RETURN cnt; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; SELECT jsonb_agg(content) INTO out_records FROM results; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off ; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb[] := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); IF fields IS NOT NULL THEN IF fields ? 'fields' THEN fields := fields->'fields'; END IF; IF fields ? 'exclude' THEN excludes=textarr(fields->'exclude'); END IF; IF fields ? 'include' THEN includes=textarr(fields->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; END IF; RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; curs = create_cursor(query); LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; IF fields IS NOT NULL THEN out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); ELSE out_records := out_records || iter_record.content; END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', array_to_json(out_records)::jsonb ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; SELECT set_version('0.4.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.2-0.4.3.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, recursion integer DEFAULT 0) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE args jsonb := j->'args'; jtype text := jsonb_typeof(j->'args'); op text := lower(j->>'op'); arg jsonb; argtext text; argstext text[] := '{}'::text[]; inobj jsonb; _numeric text := ''; ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN (%2$s)[1] AND (%2$s)[2]", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; BEGIN RAISE NOTICE 'j: %s', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RAISE NOTICE 'upper %s',jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ; RETURN cql2_query( jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ); END IF; IF j ? 'lower' THEN RETURN cql2_query( jsonb_build_object( 'op', 'lower', 'args', jsonb_build_array( j-> 'lower') ) ); END IF; IF j ? 'args' AND jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; -- END Cases where no further nesting is expected IF j ? 'op' THEN -- Special case to use JSONB index for equality IF op IN ('eq', '=') AND args->0 ? 'property' AND jsonb_typeof(args->1) IN ('number', 'string') AND (items_path(args->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(args->0->>'property')).eq, args->1); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; -- In Query - separate into separate eq statements so that we can use eq jsonb optimization IF op = 'in' THEN RAISE NOTICE '% IN args: %', repeat(' ', recursion), args; SELECT INTO inobj jsonb_agg( jsonb_build_object( 'op', 'eq', 'args', jsonb_build_array( args->0 , v) ) ) FROM jsonb_array_elements( args->1) v; RETURN cql2_query(jsonb_build_object('op','or','args',inobj)); END IF; END IF; IF j ? 'property' THEN RETURN (items_path(j->>'property')).path_txt; END IF; IF j ? 'timestamp' THEN RETURN quote_literal(j->>'timestamp'); END IF; RAISE NOTICE '%jtype: %',repeat(' ', recursion), jtype; IF jsonb_typeof(j) = 'number' THEN RETURN format('%L::numeric', j->>0); END IF; IF jsonb_typeof(j) = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jsonb_typeof(j) = 'array' THEN IF j @? '$[*] ? (@.type() == "number")' THEN RETURN CONCAT(quote_literal(textarr(j)::text), '::numeric[]'); ELSE RETURN CONCAT(quote_literal(textarr(j)::text), '::text[]'); END IF; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; RAISE NOTICE '%beforeargs op: %, args: %',repeat(' ', recursion), op, args; IF j ? 'args' THEN FOR arg in SELECT * FROM jsonb_array_elements(args) LOOP argtext := cql2_query(arg, recursion + 1); RAISE NOTICE '% -- arg: %, argtext: %', repeat(' ', recursion), arg, argtext; argstext := argstext || argtext; END LOOP; END IF; RAISE NOTICE '%afterargs op: %, argstext: %',repeat(' ', recursion), op, argstext; IF op IN ('and', 'or') THEN RAISE NOTICE 'inand op: %, argstext: %', op, argstext; SELECT concat(' ( ',array_to_string(array_agg(e), concat(' ',op,' ')),' ) ') INTO ret FROM unnest(argstext) e; RETURN ret; END IF; IF ops ? op THEN IF argstext[2] ~* 'numeric' THEN argstext := ARRAY[concat('(',argstext[1],')::numeric')] || argstext[2:3]; END IF; RETURN format(concat('(',ops->>op,')'), VARIADIC argstext); END IF; RAISE NOTICE '%op: %, argstext: %',repeat(' ', recursion), op, argstext; RETURN NULL; END; $function$ ; SELECT set_version('0.4.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.2.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '1000000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context(); CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT get_setting('context', conf); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_count(); CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE 'sql' STRICT IMMUTABLE; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_queries; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT '{items}' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p text; cursors refcursor; dstart timestamptz; dend timestamptz; step interval := '10 weeks'::interval; BEGIN IF _orderby ILIKE 'datetime d%' THEN partitions := partitions; ELSIF _orderby ILIKE 'datetime a%' THEN partitions := array_reverse(partitions); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RAISE NOTICE 'PARTITIONS ---> %',partitions; IF cardinality(partitions) > 0 THEN FOREACH p IN ARRAY partitions --EXECUTE partition_query LOOP query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s $q$, p, _where, _orderby ); RETURN NEXT query; END LOOP; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partition_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION drop_partition_constraints(IN partition text) RETURNS VOID AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, DROP CONSTRAINT IF EXISTS %I; $q$, partition, end_datetime_constraint, collections_constraint ); EXECUTE q; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_checks; CREATE OR REPLACE FUNCTION partition_checks( IN partition text, OUT min_datetime timestamptz, OUT max_datetime timestamptz, OUT min_end_datetime timestamptz, OUT max_end_datetime timestamptz, OUT collections text[], OUT cnt bigint ) RETURNS RECORD AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; q := format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime), array_agg(DISTINCT collection_id), count(*) FROM %I; $q$, partition ); EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; RAISE NOTICE '% % % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt, ftime(); IF cnt IS NULL or cnt = 0 THEN RAISE NOTICE 'Partition % is empty, removing...', partition; q := format($q$ DROP TABLE IF EXISTS %I; $q$, partition ); EXECUTE q; RETURN; END IF; RAISE NOTICE 'Running Constraint DDL %', ftime(); q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((end_datetime >= %L) AND (end_datetime <= %L)) NOT VALID, DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((collection_id = ANY(%L))) NOT VALID; $q$, partition, end_datetime_constraint, end_datetime_constraint, min_end_datetime, max_end_datetime, collections_constraint, collections_constraint, collections, partition ); RAISE NOTICE 'q: %', q; EXECUTE q; RAISE NOTICE 'Returning %', ftime(); RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION validate_constraints() RETURNS VOID AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I.%I VALIDATE CONSTRAINT %I;', current_database(), nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated IS FALSE AND nsp.nspname = 'pgstac' LOOP EXECUTE q; END LOOP; END; $$ LANGUAGE PLPGSQL; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), t[1]::timestamptz as pstart, t[2]::timestamptz as pend, est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp', 'infinity'] WHEN _indate ? 'interval' THEN textarr(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- for in, convert value, list to array syntax to match other ops IF op = 'in' and j ? 'value' and j ? 'list' THEN j := jsonb_build_array( j->'value', j->'list'); jtype := 'array'; RAISE NOTICE 'IN: j: %, jtype: %', j, jtype; END IF; IF op = 'between' and j ? 'value' and j ? 'lower' and j ? 'upper' THEN j := jsonb_build_array( j->'value', j->'lower', j->'upper'); jtype := 'array'; RAISE NOTICE 'BETWEEN: j: %, jtype: %', j, jtype; END IF; IF op = 'not' AND jtype = 'object' THEN j := jsonb_build_array( j ); jtype := 'array'; RAISE NOTICE 'NOT: j: %, jtype: %', j, jtype; END IF; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ DROP FUNCTION IF EXISTS cql2_query; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, recursion int DEFAULT 0) RETURNS text AS $$ DECLARE args jsonb := j->'args'; jtype text := jsonb_typeof(j->'args'); op text := lower(j->>'op'); arg jsonb; argtext text; argstext text[] := '{}'::text[]; inobj jsonb; _numeric text := ''; ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN (%2$s)[1] AND (%2$s)[2]", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; BEGIN RAISE NOTICE 'j: %s', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RAISE NOTICE 'upper %s',jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ; RETURN cql2_query( jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ); END IF; IF j ? 'lower' THEN RETURN cql2_query( jsonb_build_object( 'op', 'lower', 'args', jsonb_build_array( j-> 'lower') ) ); END IF; IF j ? 'args' AND jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; -- END Cases where no further nesting is expected IF j ? 'op' THEN -- Special case to use JSONB index for equality IF op = 'eq' AND args->0 ? 'property' AND jsonb_typeof(args->1) IN ('number', 'string') AND (items_path(args->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(args->0->>'property')).eq, args->1); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; -- In Query - separate into separate eq statements so that we can use eq jsonb optimization IF op = 'in' THEN RAISE NOTICE '% IN args: %', repeat(' ', recursion), args; SELECT INTO inobj jsonb_agg( jsonb_build_object( 'op', 'eq', 'args', jsonb_build_array( args->0 , v) ) ) FROM jsonb_array_elements( args->1) v; RETURN cql2_query(jsonb_build_object('op','or','args',inobj)); END IF; END IF; IF j ? 'property' THEN RETURN (items_path(j->>'property')).path_txt; END IF; IF j ? 'timestamp' THEN RETURN quote_literal(j->>'timestamp'); END IF; RAISE NOTICE '%jtype: %',repeat(' ', recursion), jtype; IF jsonb_typeof(j) = 'number' THEN RETURN format('%L::numeric', j->>0); END IF; IF jsonb_typeof(j) = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jsonb_typeof(j) = 'array' THEN IF j @? '$[*] ? (@.type() == "number")' THEN RETURN CONCAT(quote_literal(textarr(j)::text), '::numeric[]'); ELSE RETURN CONCAT(quote_literal(textarr(j)::text), '::text[]'); END IF; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; RAISE NOTICE '%beforeargs op: %, args: %',repeat(' ', recursion), op, args; IF j ? 'args' THEN FOR arg in SELECT * FROM jsonb_array_elements(args) LOOP argtext := cql2_query(arg, recursion + 1); RAISE NOTICE '% -- arg: %, argtext: %', repeat(' ', recursion), arg, argtext; argstext := argstext || argtext; END LOOP; END IF; RAISE NOTICE '%afterargs op: %, argstext: %',repeat(' ', recursion), op, argstext; IF op IN ('and', 'or') THEN RAISE NOTICE 'inand op: %, argstext: %', op, argstext; SELECT concat(' ( ',array_to_string(array_agg(e), concat(' ',op,' ')),' ) ') INTO ret FROM unnest(argstext) e; RETURN ret; END IF; IF ops ? op THEN IF argstext[2] ~* 'numeric' THEN argstext := ARRAY[concat('(',argstext[1],')::numeric')] || argstext[2:3]; END IF; RETURN format(concat('(',ops->>op,')'), VARIADIC argstext); END IF; RAISE NOTICE '%op: %, argstext: %',repeat(' ', recursion), op, argstext; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE filterlang text; search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL Final: %', search; filterlang := COALESCE( search->>'filter-lang', get_setting('default-filter-lang', _search->'conf') ); IF filterlang = 'cql-json' THEN search := query_to_cqlfilter(search); search := add_filters_to_cql(search); _where := cql_query_op(search->'filter'); ELSE _where := cql2_query(search->'filter'); END IF; IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION field_orderby(p text) RETURNS text AS $$ WITH t AS ( SELECT replace(trim(substring(indexdef from 'btree \((.*)\)')),' ','')as s FROM pg_indexes WHERE schemaname='pgstac' AND tablename='items' AND indexdef ~* 'btree' AND indexdef ~* 'properties' ) SELECT s FROM t WHERE strpos(s, lower(trim(p)))>0; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce(field_orderby((items_path(value->>'field')).path_txt), (items_path(value->>'field')).path) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(conf), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(conf), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl(conf) OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context(conf) = 'on' OR ( context(conf) = 'auto' AND ( sw.estimated_count < context_estimated_count(conf) OR sw.estimated_cost < context_estimated_cost(conf) ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; CREATE OR REPLACE FUNCTION items_count(_where text) RETURNS bigint AS $$ DECLARE cnt bigint; BEGIN EXECUTE format('SELECT count(*) FROM items WHERE %s', _where) INTO cnt; RETURN cnt; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; SELECT jsonb_agg(content) INTO out_records FROM results; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off ; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb[] := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); IF fields IS NOT NULL THEN IF fields ? 'fields' THEN fields := fields->'fields'; END IF; IF fields ? 'exclude' THEN excludes=textarr(fields->'exclude'); END IF; IF fields ? 'include' THEN includes=textarr(fields->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; END IF; RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; curs = create_cursor(query); LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; IF fields IS NOT NULL THEN out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); ELSE out_records := out_records || iter_record.content; END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', array_to_json(out_records)::jsonb ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; SELECT set_version('0.4.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.3-0.4.4.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.base_stac_query(j jsonb) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE _where text := ''; dtrange tstzrange; geom geometry; BEGIN IF j ? 'ids' THEN _where := format('%s AND id = ANY (%L) ', _where, textarr(j->'ids')); END IF; IF j ? 'collections' THEN _where := format('%s AND collection_id = ANY (%L) ', _where, textarr(j->'collections')); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); _where := format('%s AND datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', _where, upper(dtrange), lower(dtrange) ); END IF; IF j ? 'bbox' THEN geom := bbox_geom(j->'bbox'); _where := format('%s AND geometry && %L', _where, geom ); END IF; IF j ? 'intersects' THEN geom := st_fromgeojson(j->>'intersects'); _where := format('%s AND st_intersects(geometry, %L)', _where, geom ); END IF; RETURN _where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.item_by_id(_id text) RETURNS pgstac.items LANGUAGE plpgsql AS $function$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id LIMIT 1; RETURN i; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.cql_to_where(_search jsonb DEFAULT '{}'::jsonb) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE filterlang text; search jsonb := _search; base_where text; _where text; BEGIN RAISE NOTICE 'SEARCH CQL Final: %', search; filterlang := COALESCE( search->>'filter-lang', get_setting('default-filter-lang', _search->'conf') ); base_where := base_stac_query(search); IF filterlang = 'cql-json' THEN search := query_to_cqlfilter(search); -- search := add_filters_to_cql(search); _where := cql_query_op(search->'filter'); ELSE _where := cql2_query(search->'filter'); END IF; IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN format('( %s ) %s ', _where, base_where); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_token_filter(_search jsonb DEFAULT '{}'::jsonb, token_rec jsonb DEFAULT NULL::jsonb) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM item_by_id(token_id) as items; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SET jit TO 'off' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN FOR id IN SELECT jsonb_array_elements_text(_search->'ids') LOOP INSERT INTO results (content) SELECT content FROM item_by_id(id); END LOOP; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) RETURNS pgstac.search_wheres LANGUAGE plpgsql AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(conf), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(conf), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl(conf) OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context(conf) = 'on' OR ( context(conf) = 'auto' AND ( sw.estimated_count < context_estimated_count(conf) OR sw.estimated_cost < context_estimated_cost(conf) ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $function$ ; SELECT set_version('0.4.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.3.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '1000000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context(); CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT get_setting('context', conf); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_count(); CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE 'sql' STRICT IMMUTABLE; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_queries; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT '{items}' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p text; cursors refcursor; dstart timestamptz; dend timestamptz; step interval := '10 weeks'::interval; BEGIN IF _orderby ILIKE 'datetime d%' THEN partitions := partitions; ELSIF _orderby ILIKE 'datetime a%' THEN partitions := array_reverse(partitions); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RAISE NOTICE 'PARTITIONS ---> %',partitions; IF cardinality(partitions) > 0 THEN FOREACH p IN ARRAY partitions --EXECUTE partition_query LOOP query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s $q$, p, _where, _orderby ); RETURN NEXT query; END LOOP; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partition_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION drop_partition_constraints(IN partition text) RETURNS VOID AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, DROP CONSTRAINT IF EXISTS %I; $q$, partition, end_datetime_constraint, collections_constraint ); EXECUTE q; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_checks; CREATE OR REPLACE FUNCTION partition_checks( IN partition text, OUT min_datetime timestamptz, OUT max_datetime timestamptz, OUT min_end_datetime timestamptz, OUT max_end_datetime timestamptz, OUT collections text[], OUT cnt bigint ) RETURNS RECORD AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; q := format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime), array_agg(DISTINCT collection_id), count(*) FROM %I; $q$, partition ); EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; RAISE NOTICE '% % % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt, ftime(); IF cnt IS NULL or cnt = 0 THEN RAISE NOTICE 'Partition % is empty, removing...', partition; q := format($q$ DROP TABLE IF EXISTS %I; $q$, partition ); EXECUTE q; RETURN; END IF; RAISE NOTICE 'Running Constraint DDL %', ftime(); q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((end_datetime >= %L) AND (end_datetime <= %L)) NOT VALID, DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((collection_id = ANY(%L))) NOT VALID; $q$, partition, end_datetime_constraint, end_datetime_constraint, min_end_datetime, max_end_datetime, collections_constraint, collections_constraint, collections, partition ); RAISE NOTICE 'q: %', q; EXECUTE q; RAISE NOTICE 'Returning %', ftime(); RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION validate_constraints() RETURNS VOID AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I.%I VALIDATE CONSTRAINT %I;', current_database(), nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated IS FALSE AND nsp.nspname = 'pgstac' LOOP EXECUTE q; END LOOP; END; $$ LANGUAGE PLPGSQL; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), t[1]::timestamptz as pstart, t[2]::timestamptz as pend, est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp', 'infinity'] WHEN _indate ? 'interval' THEN textarr(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- for in, convert value, list to array syntax to match other ops IF op = 'in' and j ? 'value' and j ? 'list' THEN j := jsonb_build_array( j->'value', j->'list'); jtype := 'array'; RAISE NOTICE 'IN: j: %, jtype: %', j, jtype; END IF; IF op = 'between' and j ? 'value' and j ? 'lower' and j ? 'upper' THEN j := jsonb_build_array( j->'value', j->'lower', j->'upper'); jtype := 'array'; RAISE NOTICE 'BETWEEN: j: %, jtype: %', j, jtype; END IF; IF op = 'not' AND jtype = 'object' THEN j := jsonb_build_array( j ); jtype := 'array'; RAISE NOTICE 'NOT: j: %, jtype: %', j, jtype; END IF; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ DROP FUNCTION IF EXISTS cql2_query; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, recursion int DEFAULT 0) RETURNS text AS $$ DECLARE args jsonb := j->'args'; jtype text := jsonb_typeof(j->'args'); op text := lower(j->>'op'); arg jsonb; argtext text; argstext text[] := '{}'::text[]; inobj jsonb; _numeric text := ''; ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN (%2$s)[1] AND (%2$s)[2]", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; BEGIN RAISE NOTICE 'j: %s', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RAISE NOTICE 'upper %s',jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ; RETURN cql2_query( jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ); END IF; IF j ? 'lower' THEN RETURN cql2_query( jsonb_build_object( 'op', 'lower', 'args', jsonb_build_array( j-> 'lower') ) ); END IF; IF j ? 'args' AND jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; -- END Cases where no further nesting is expected IF j ? 'op' THEN -- Special case to use JSONB index for equality IF op IN ('eq', '=') AND args->0 ? 'property' AND jsonb_typeof(args->1) IN ('number', 'string') AND (items_path(args->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(args->0->>'property')).eq, args->1); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; -- In Query - separate into separate eq statements so that we can use eq jsonb optimization IF op = 'in' THEN RAISE NOTICE '% IN args: %', repeat(' ', recursion), args; SELECT INTO inobj jsonb_agg( jsonb_build_object( 'op', 'eq', 'args', jsonb_build_array( args->0 , v) ) ) FROM jsonb_array_elements( args->1) v; RETURN cql2_query(jsonb_build_object('op','or','args',inobj)); END IF; END IF; IF j ? 'property' THEN RETURN (items_path(j->>'property')).path_txt; END IF; IF j ? 'timestamp' THEN RETURN quote_literal(j->>'timestamp'); END IF; RAISE NOTICE '%jtype: %',repeat(' ', recursion), jtype; IF jsonb_typeof(j) = 'number' THEN RETURN format('%L::numeric', j->>0); END IF; IF jsonb_typeof(j) = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jsonb_typeof(j) = 'array' THEN IF j @? '$[*] ? (@.type() == "number")' THEN RETURN CONCAT(quote_literal(textarr(j)::text), '::numeric[]'); ELSE RETURN CONCAT(quote_literal(textarr(j)::text), '::text[]'); END IF; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; RAISE NOTICE '%beforeargs op: %, args: %',repeat(' ', recursion), op, args; IF j ? 'args' THEN FOR arg in SELECT * FROM jsonb_array_elements(args) LOOP argtext := cql2_query(arg, recursion + 1); RAISE NOTICE '% -- arg: %, argtext: %', repeat(' ', recursion), arg, argtext; argstext := argstext || argtext; END LOOP; END IF; RAISE NOTICE '%afterargs op: %, argstext: %',repeat(' ', recursion), op, argstext; IF op IN ('and', 'or') THEN RAISE NOTICE 'inand op: %, argstext: %', op, argstext; SELECT concat(' ( ',array_to_string(array_agg(e), concat(' ',op,' ')),' ) ') INTO ret FROM unnest(argstext) e; RETURN ret; END IF; IF ops ? op THEN IF argstext[2] ~* 'numeric' THEN argstext := ARRAY[concat('(',argstext[1],')::numeric')] || argstext[2:3]; END IF; RETURN format(concat('(',ops->>op,')'), VARIADIC argstext); END IF; RAISE NOTICE '%op: %, argstext: %',repeat(' ', recursion), op, argstext; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE filterlang text; search jsonb := _search; _where text; BEGIN RAISE NOTICE 'SEARCH CQL Final: %', search; filterlang := COALESCE( search->>'filter-lang', get_setting('default-filter-lang', _search->'conf') ); IF filterlang = 'cql-json' THEN search := query_to_cqlfilter(search); search := add_filters_to_cql(search); _where := cql_query_op(search->'filter'); ELSE _where := cql2_query(search->'filter'); END IF; IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION field_orderby(p text) RETURNS text AS $$ WITH t AS ( SELECT replace(trim(substring(indexdef from 'btree \((.*)\)')),' ','')as s FROM pg_indexes WHERE schemaname='pgstac' AND tablename='items' AND indexdef ~* 'btree' AND indexdef ~* 'properties' ) SELECT s FROM t WHERE strpos(s, lower(trim(p)))>0; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce(field_orderby((items_path(value->>'field')).path_txt), (items_path(value->>'field')).path) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(conf), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(conf), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl(conf) OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context(conf) = 'on' OR ( context(conf) = 'auto' AND ( sw.estimated_count < context_estimated_count(conf) OR sw.estimated_cost < context_estimated_cost(conf) ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; CREATE OR REPLACE FUNCTION items_count(_where text) RETURNS bigint AS $$ DECLARE cnt bigint; BEGIN EXECUTE format('SELECT count(*) FROM items WHERE %s', _where) INTO cnt; RETURN cnt; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; SELECT jsonb_agg(content) INTO out_records FROM results; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off ; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb[] := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); IF fields IS NOT NULL THEN IF fields ? 'fields' THEN fields := fields->'fields'; END IF; IF fields ? 'exclude' THEN excludes=textarr(fields->'exclude'); END IF; IF fields ? 'include' THEN includes=textarr(fields->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; END IF; RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; curs = create_cursor(query); LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; IF fields IS NOT NULL THEN out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); ELSE out_records := out_records || iter_record.content; END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', array_to_json(out_records)::jsonb ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; SELECT set_version('0.4.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.4-0.4.5.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.base_stac_query(j jsonb) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE _where text := ''; dtrange tstzrange; geom geometry; BEGIN IF j ? 'ids' THEN _where := format('%s AND id = ANY (%L) ', _where, textarr(j->'ids')); END IF; IF j ? 'collections' THEN _where := format('%s AND collection_id = ANY (%L) ', _where, textarr(j->'collections')); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); _where := format('%s AND datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', _where, upper(dtrange), lower(dtrange) ); END IF; IF j ? 'bbox' THEN geom := bbox_geom(j->'bbox'); _where := format('%s AND geometry && %L', _where, geom ); END IF; IF j ? 'intersects' THEN geom := st_geomfromgeojson(j->>'intersects'); _where := format('%s AND st_intersects(geometry, %L)', _where, geom ); END IF; RETURN _where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_ignore_insert_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SET jit TO 'off' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN FOR id IN SELECT jsonb_array_elements_text(_search->'ids') LOOP INSERT INTO results (content) SELECT content FROM item_by_id(id) WHERE content IS NOT NULL; END LOOP; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; SELECT set_version('0.4.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.4.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '1000000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context(); CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT get_setting('context', conf); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_count(); CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE 'sql' STRICT IMMUTABLE; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_queries; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT '{items}' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p text; cursors refcursor; dstart timestamptz; dend timestamptz; step interval := '10 weeks'::interval; BEGIN IF _orderby ILIKE 'datetime d%' THEN partitions := partitions; ELSIF _orderby ILIKE 'datetime a%' THEN partitions := array_reverse(partitions); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RAISE NOTICE 'PARTITIONS ---> %',partitions; IF cardinality(partitions) > 0 THEN FOREACH p IN ARRAY partitions --EXECUTE partition_query LOOP query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s $q$, p, _where, _orderby ); RETURN NEXT query; END LOOP; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partition_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION drop_partition_constraints(IN partition text) RETURNS VOID AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, DROP CONSTRAINT IF EXISTS %I; $q$, partition, end_datetime_constraint, collections_constraint ); EXECUTE q; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_checks; CREATE OR REPLACE FUNCTION partition_checks( IN partition text, OUT min_datetime timestamptz, OUT max_datetime timestamptz, OUT min_end_datetime timestamptz, OUT max_end_datetime timestamptz, OUT collections text[], OUT cnt bigint ) RETURNS RECORD AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; q := format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime), array_agg(DISTINCT collection_id), count(*) FROM %I; $q$, partition ); EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; RAISE NOTICE '% % % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt, ftime(); IF cnt IS NULL or cnt = 0 THEN RAISE NOTICE 'Partition % is empty, removing...', partition; q := format($q$ DROP TABLE IF EXISTS %I; $q$, partition ); EXECUTE q; RETURN; END IF; RAISE NOTICE 'Running Constraint DDL %', ftime(); q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((end_datetime >= %L) AND (end_datetime <= %L)) NOT VALID, DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((collection_id = ANY(%L))) NOT VALID; $q$, partition, end_datetime_constraint, end_datetime_constraint, min_end_datetime, max_end_datetime, collections_constraint, collections_constraint, collections, partition ); RAISE NOTICE 'q: %', q; EXECUTE q; RAISE NOTICE 'Returning %', ftime(); RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION validate_constraints() RETURNS VOID AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I.%I VALIDATE CONSTRAINT %I;', current_database(), nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated IS FALSE AND nsp.nspname = 'pgstac' LOOP EXECUTE q; END LOOP; END; $$ LANGUAGE PLPGSQL; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), t[1]::timestamptz as pstart, t[2]::timestamptz as pend, est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION item_by_id(_id text) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp', 'infinity'] WHEN _indate ? 'interval' THEN textarr(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_stac_query(j jsonb) RETURNS text AS $$ DECLARE _where text := ''; dtrange tstzrange; geom geometry; BEGIN IF j ? 'ids' THEN _where := format('%s AND id = ANY (%L) ', _where, textarr(j->'ids')); END IF; IF j ? 'collections' THEN _where := format('%s AND collection_id = ANY (%L) ', _where, textarr(j->'collections')); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); _where := format('%s AND datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', _where, upper(dtrange), lower(dtrange) ); END IF; IF j ? 'bbox' THEN geom := bbox_geom(j->'bbox'); _where := format('%s AND geometry && %L', _where, geom ); END IF; IF j ? 'intersects' THEN geom := st_fromgeojson(j->>'intersects'); _where := format('%s AND st_intersects(geometry, %L)', _where, geom ); END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- for in, convert value, list to array syntax to match other ops IF op = 'in' and j ? 'value' and j ? 'list' THEN j := jsonb_build_array( j->'value', j->'list'); jtype := 'array'; RAISE NOTICE 'IN: j: %, jtype: %', j, jtype; END IF; IF op = 'between' and j ? 'value' and j ? 'lower' and j ? 'upper' THEN j := jsonb_build_array( j->'value', j->'lower', j->'upper'); jtype := 'array'; RAISE NOTICE 'BETWEEN: j: %, jtype: %', j, jtype; END IF; IF op = 'not' AND jtype = 'object' THEN j := jsonb_build_array( j ); jtype := 'array'; RAISE NOTICE 'NOT: j: %, jtype: %', j, jtype; END IF; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ DROP FUNCTION IF EXISTS cql2_query; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, recursion int DEFAULT 0) RETURNS text AS $$ DECLARE args jsonb := j->'args'; jtype text := jsonb_typeof(j->'args'); op text := lower(j->>'op'); arg jsonb; argtext text; argstext text[] := '{}'::text[]; inobj jsonb; _numeric text := ''; ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN (%2$s)[1] AND (%2$s)[2]", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; BEGIN RAISE NOTICE 'j: %s', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RAISE NOTICE 'upper %s',jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ; RETURN cql2_query( jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ); END IF; IF j ? 'lower' THEN RETURN cql2_query( jsonb_build_object( 'op', 'lower', 'args', jsonb_build_array( j-> 'lower') ) ); END IF; IF j ? 'args' AND jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; -- END Cases where no further nesting is expected IF j ? 'op' THEN -- Special case to use JSONB index for equality IF op IN ('eq', '=') AND args->0 ? 'property' AND jsonb_typeof(args->1) IN ('number', 'string') AND (items_path(args->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(args->0->>'property')).eq, args->1); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; -- In Query - separate into separate eq statements so that we can use eq jsonb optimization IF op = 'in' THEN RAISE NOTICE '% IN args: %', repeat(' ', recursion), args; SELECT INTO inobj jsonb_agg( jsonb_build_object( 'op', 'eq', 'args', jsonb_build_array( args->0 , v) ) ) FROM jsonb_array_elements( args->1) v; RETURN cql2_query(jsonb_build_object('op','or','args',inobj)); END IF; END IF; IF j ? 'property' THEN RETURN (items_path(j->>'property')).path_txt; END IF; IF j ? 'timestamp' THEN RETURN quote_literal(j->>'timestamp'); END IF; RAISE NOTICE '%jtype: %',repeat(' ', recursion), jtype; IF jsonb_typeof(j) = 'number' THEN RETURN format('%L::numeric', j->>0); END IF; IF jsonb_typeof(j) = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jsonb_typeof(j) = 'array' THEN IF j @? '$[*] ? (@.type() == "number")' THEN RETURN CONCAT(quote_literal(textarr(j)::text), '::numeric[]'); ELSE RETURN CONCAT(quote_literal(textarr(j)::text), '::text[]'); END IF; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; RAISE NOTICE '%beforeargs op: %, args: %',repeat(' ', recursion), op, args; IF j ? 'args' THEN FOR arg in SELECT * FROM jsonb_array_elements(args) LOOP argtext := cql2_query(arg, recursion + 1); RAISE NOTICE '% -- arg: %, argtext: %', repeat(' ', recursion), arg, argtext; argstext := argstext || argtext; END LOOP; END IF; RAISE NOTICE '%afterargs op: %, argstext: %',repeat(' ', recursion), op, argstext; IF op IN ('and', 'or') THEN RAISE NOTICE 'inand op: %, argstext: %', op, argstext; SELECT concat(' ( ',array_to_string(array_agg(e), concat(' ',op,' ')),' ) ') INTO ret FROM unnest(argstext) e; RETURN ret; END IF; IF ops ? op THEN IF argstext[2] ~* 'numeric' THEN argstext := ARRAY[concat('(',argstext[1],')::numeric')] || argstext[2:3]; END IF; RETURN format(concat('(',ops->>op,')'), VARIADIC argstext); END IF; RAISE NOTICE '%op: %, argstext: %',repeat(' ', recursion), op, argstext; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE filterlang text; search jsonb := _search; base_where text; _where text; BEGIN RAISE NOTICE 'SEARCH CQL Final: %', search; filterlang := COALESCE( search->>'filter-lang', get_setting('default-filter-lang', _search->'conf') ); base_where := base_stac_query(search); IF filterlang = 'cql-json' THEN search := query_to_cqlfilter(search); -- search := add_filters_to_cql(search); _where := cql_query_op(search->'filter'); ELSE _where := cql2_query(search->'filter'); END IF; IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN format('( %s ) %s ', _where, base_where); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION field_orderby(p text) RETURNS text AS $$ WITH t AS ( SELECT replace(trim(substring(indexdef from 'btree \((.*)\)')),' ','')as s FROM pg_indexes WHERE schemaname='pgstac' AND tablename='items' AND indexdef ~* 'btree' AND indexdef ~* 'properties' ) SELECT s FROM t WHERE strpos(s, lower(trim(p)))>0; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce(field_orderby((items_path(value->>'field')).path_txt), (items_path(value->>'field')).path) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM item_by_id(token_id) as items; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(conf), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(conf), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl(conf) OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context(conf) = 'on' OR ( context(conf) = 'auto' AND ( sw.estimated_count < context_estimated_count(conf) OR sw.estimated_cost < context_estimated_cost(conf) ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; CREATE OR REPLACE FUNCTION items_count(_where text) RETURNS bigint AS $$ DECLARE cnt bigint; BEGIN EXECUTE format('SELECT count(*) FROM items WHERE %s', _where) INTO cnt; RETURN cnt; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN FOR id IN SELECT jsonb_array_elements_text(_search->'ids') LOOP INSERT INTO results (content) SELECT content FROM item_by_id(id); END LOOP; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off ; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb[] := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); IF fields IS NOT NULL THEN IF fields ? 'fields' THEN fields := fields->'fields'; END IF; IF fields ? 'exclude' THEN excludes=textarr(fields->'exclude'); END IF; IF fields ? 'include' THEN includes=textarr(fields->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; END IF; RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; curs = create_cursor(query); LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; IF fields IS NOT NULL THEN out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); ELSE out_records := out_records || iter_record.content; END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', array_to_json(out_records)::jsonb ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; SELECT set_version('0.4.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.5-0.5.0.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; ALTER SCHEMA pgstac rename to pgstac_045; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT $1->>0; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props ? 'end_datetime' THEN dt := props->'start_datetime'; edt := props->'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->'datetime'; edt := props->'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE EXCEPTION 'Either datetime or both start_datetime and end_datetime must be set.'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id', 'links', '[]'::jsonb ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id), name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('in', '%s = ANY (%s)', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN (%2$s)[1] AND (%2$s)[2]', NULL), ('isnull', '%s IS NULL', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template; ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN format('upper(%s)', cql2_query(j->'upper')); END IF; IF j ? 'lower' THEN RETURN format('lower(%s)', cql2_query(j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RETURN format( '%s = ANY (%L)', cql2_query(args->0), to_text_array(args->1) ); END IF; IF op = 'between' THEN SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; RETURN format( '%s BETWEEN %s and %s', cql2_query(args->0, wrapper), cql2_query(args->1->0, wrapper), cql2_query(args->1->1, wrapper) ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF j ? 'property' THEN RETURN (queryable(j->>'property')).expression; END IF; IF wrapper IS NOT NULL THEN EXECUTE format('SELECT %I(%L)', wrapper, j) INTO literal; RAISE NOTICE '% % %',wrapper, j, literal; RETURN format('%I(%L)', wrapper, j); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb, _collection jsonb) RETURNS jsonb AS $$ SELECT jsonb_object_agg( key, CASE WHEN jsonb_typeof(c.value) = 'object' AND jsonb_typeof(i.value) = 'object' THEN content_slim(i.value, c.value) ELSE i.value END ) FROM jsonb_each(_item) as i LEFT JOIN jsonb_each(_collection) as c USING (key) WHERE i.value IS DISTINCT FROM c.value ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT content_slim(_item - '{id,type,collection,geometry,bbox}'::text[], collection_base_item(_item->>'collection')); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := coalesce(fields->'includes', fields->'include', '[]'::jsonb); excludes jsonb := coalesce(fields->'excludes', fields->'exclude', '[]'::jsonb); BEGIN IF f IS NULL THEN RETURN NULL; ELSIF jsonb_array_length(includes)>0 AND includes ? f THEN RETURN TRUE; ELSIF jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; ELSIF jsonb_array_length(includes)>0 AND NOT includes ? f THEN RETURN FALSE; END IF; RETURN TRUE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION key_filter(IN k text, IN val jsonb, INOUT kf jsonb, OUT include boolean) AS $$ DECLARE includes jsonb := coalesce(kf->'includes', kf->'include', '[]'::jsonb); excludes jsonb := coalesce(kf->'excludes', kf->'exclude', '[]'::jsonb); BEGIN RAISE NOTICE '% % %', k, val, kf; include := TRUE; IF k = 'properties' AND NOT excludes ? 'properties' THEN excludes := excludes || '["properties"]'; include := TRUE; RAISE NOTICE 'Prop include %', include; ELSIF jsonb_array_length(excludes)>0 AND excludes ? k THEN include := FALSE; ELSIF jsonb_array_length(includes)>0 AND NOT includes ? k THEN include := FALSE; ELSIF jsonb_array_length(includes)>0 AND includes ? k THEN includes := '[]'::jsonb; RAISE NOTICE 'KF: %', kf; END IF; kf := jsonb_build_object('includes', includes, 'excludes', excludes); RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION strip_assets(a jsonb) RETURNS jsonb AS $$ WITH t AS (SELECT * FROM jsonb_each(a)) SELECT jsonb_object_agg(key, value) FROM t WHERE value ? 'href'; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _collection jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT jsonb_strip_nulls(jsonb_object_agg( key, CASE WHEN key = 'properties' AND include_field('properties', fields) THEN i.value WHEN key = 'properties' THEN content_hydrate(i.value, c.value, kf) WHEN c.value IS NULL AND key != 'properties' THEN i.value WHEN key = 'assets' AND jsonb_typeof(c.value) = 'object' AND jsonb_typeof(i.value) = 'object' THEN strip_assets(content_hydrate(i.value, c.value, kf)) WHEN jsonb_typeof(c.value) = 'object' AND jsonb_typeof(i.value) = 'object' THEN content_hydrate(i.value, c.value, kf) ELSE coalesce(i.value, c.value) END )) FROM jsonb_each(coalesce(_item,'{}'::jsonb)) as i FULL JOIN jsonb_each(coalesce(_collection,'{}'::jsonb)) as c USING (key) JOIN LATERAL ( SELECT kf, include FROM key_filter(key, i.value, fields) ) as k ON (include) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry)::jsonb; END IF; IF include_field('bbox', fields) THEN bbox := geom_bbox(_item.geometry)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'bbox',bbox, 'collection', _item.collection ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL STABLE; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN SELECT delete_item(content->>'id', content->>'collection'); SELECT create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT content_hydrate(items, _search->'fields') FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := content_hydrate(iter_record, _search->'fields'); IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(content_dehydrate(first_record)))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; RESET ROLE; INSERT INTO pgstac.collections (content) SELECT content FROM pgstac_045.collections; INSERT INTO pgstac.items_staging SELECT content FROM pgstac_045.items; SELECT set_version('0.5.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.4.5.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE SCHEMA IF NOT EXISTS pgstac; SET SEARCH_PATH TO pgstac, public; CREATE TABLE migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '1000000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context(); CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT get_setting('context', conf); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_count(); CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION estimated_count(_where text) RETURNS bigint AS $$ DECLARE rec record; rows bigint; BEGIN FOR rec in EXECUTE format( $q$ EXPLAIN SELECT 1 FROM items WHERE %s $q$, _where) LOOP rows := substring(rec."QUERY PLAN" FROM ' rows=([[:digit:]]+)'); EXIT WHEN rows IS NOT NULL; END LOOP; RETURN rows; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE 'sql' STRICT IMMUTABLE; /* converts a jsonb text array to a pg text[] array */ CREATE OR REPLACE FUNCTION textarr(_js jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof(_js) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text(_js)) ELSE ARRAY[_js->>0] END ; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) select * from extract_all; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION jsonb_obj_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, value FROM jsonb_each(jdata) union all select path || obj_key, obj_value from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' where obj_key is not null ) select * from extract_all; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_val_paths (IN jdata jsonb, OUT path text[], OUT value jsonb) RETURNS SETOF RECORD AS $$ SELECT * FROM jsonb_obj_paths(jdata) WHERE jsonb_typeof(value) not in ('object','array'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_includes(IN path text[], IN includes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(includes) i) SELECT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(i), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION path_excludes(IN path text[], IN excludes text[]) RETURNS BOOLEAN AS $$ WITH t AS (SELECT unnest(excludes) e) SELECT NOT EXISTS ( SELECT 1 FROM t WHERE path @> string_to_array(trim(e), '.') ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_obj_paths_filtered ( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[], OUT path text[], OUT value jsonb ) RETURNS SETOF RECORD AS $$ SELECT path, value FROM jsonb_obj_paths(jdata) WHERE CASE WHEN cardinality(includes) > 0 THEN path_includes(path, includes) ELSE TRUE END AND path_excludes(path, excludes) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION filter_jsonb( IN jdata jsonb, IN includes text[] DEFAULT ARRAY[]::text[], IN excludes text[] DEFAULT ARRAY[]::text[] ) RETURNS jsonb AS $$ DECLARE rec RECORD; outj jsonb := '{}'::jsonb; created_paths text[] := '{}'::text[]; BEGIN IF empty_arr(includes) AND empty_arr(excludes) THEN RAISE NOTICE 'no filter'; RETURN jdata; END IF; FOR rec in SELECT * FROM jsonb_obj_paths_filtered(jdata, includes, excludes) WHERE jsonb_typeof(value) != 'object' LOOP IF array_length(rec.path,1)>1 THEN FOR i IN 1..(array_length(rec.path,1)-1) LOOP IF NOT array_to_string(rec.path[1:i],'.') = ANY (created_paths) THEN outj := jsonb_set(outj, rec.path[1:i],'{}', true); created_paths := created_paths || array_to_string(rec.path[1:i],'.'); END IF; END LOOP; END IF; outj := jsonb_set(outj, rec.path, rec.value, true); created_paths := created_paths || array_to_string(rec.path,'.'); END LOOP; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* Functions to create an iterable of cursors over partitions. */ CREATE OR REPLACE FUNCTION create_cursor(q text) RETURNS refcursor AS $$ DECLARE curs refcursor; BEGIN OPEN curs FOR EXECUTE q; RETURN curs; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_queries; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT '{items}' ) RETURNS SETOF text AS $$ DECLARE partition_query text; query text; p text; cursors refcursor; dstart timestamptz; dend timestamptz; step interval := '10 weeks'::interval; BEGIN IF _orderby ILIKE 'datetime d%' THEN partitions := partitions; ELSIF _orderby ILIKE 'datetime a%' THEN partitions := array_reverse(partitions); ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RAISE NOTICE 'PARTITIONS ---> %',partitions; IF cardinality(partitions) > 0 THEN FOREACH p IN ARRAY partitions --EXECUTE partition_query LOOP query := format($q$ SELECT * FROM %I WHERE %s ORDER BY %s $q$, p, _where, _orderby ); RETURN NEXT query; END LOOP; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_cursor( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC' ) RETURNS SETOF refcursor AS $$ DECLARE partition_query text; query text; p record; cursors refcursor; BEGIN FOR query IN SELECT * FROM partition_queries(_where, _orderby) LOOP RETURN NEXT create_cursor(query); END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_count( IN _where text DEFAULT 'TRUE' ) RETURNS bigint AS $$ DECLARE partition_query text; query text; p record; subtotal bigint; total bigint := 0; BEGIN partition_query := format($q$ SELECT partition, tstzrange FROM items_partitions ORDER BY tstzrange DESC; $q$); RAISE NOTICE 'Partition Query: %', partition_query; FOR p IN EXECUTE partition_query LOOP query := format($q$ SELECT count(*) FROM items WHERE datetime BETWEEN %L AND %L AND %s $q$, lower(p.tstzrange), upper(p.tstzrange), _where ); RAISE NOTICE 'Query %', query; RAISE NOTICE 'Partition %, Count %, Total %',p.partition, subtotal, total; EXECUTE query INTO subtotal; total := subtotal + total; END LOOP; RETURN total; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION drop_partition_constraints(IN partition text) RETURNS VOID AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, DROP CONSTRAINT IF EXISTS %I; $q$, partition, end_datetime_constraint, collections_constraint ); EXECUTE q; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS partition_checks; CREATE OR REPLACE FUNCTION partition_checks( IN partition text, OUT min_datetime timestamptz, OUT max_datetime timestamptz, OUT min_end_datetime timestamptz, OUT max_end_datetime timestamptz, OUT collections text[], OUT cnt bigint ) RETURNS RECORD AS $$ DECLARE q text; end_datetime_constraint text := concat(partition, '_end_datetime_constraint'); collections_constraint text := concat(partition, '_collections_constraint'); BEGIN RAISE NOTICE 'CREATING CONSTRAINTS FOR %', partition; q := format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime), array_agg(DISTINCT collection_id), count(*) FROM %I; $q$, partition ); EXECUTE q INTO min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt; RAISE NOTICE '% % % % % % %', min_datetime, max_datetime, min_end_datetime, max_end_datetime, collections, cnt, ftime(); IF cnt IS NULL or cnt = 0 THEN RAISE NOTICE 'Partition % is empty, removing...', partition; q := format($q$ DROP TABLE IF EXISTS %I; $q$, partition ); EXECUTE q; RETURN; END IF; RAISE NOTICE 'Running Constraint DDL %', ftime(); q := format($q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((end_datetime >= %L) AND (end_datetime <= %L)) NOT VALID, DROP CONSTRAINT IF EXISTS %I, ADD CONSTRAINT %I check((collection_id = ANY(%L))) NOT VALID; $q$, partition, end_datetime_constraint, end_datetime_constraint, min_end_datetime, max_end_datetime, collections_constraint, collections_constraint, collections, partition ); RAISE NOTICE 'q: %', q; EXECUTE q; RAISE NOTICE 'Returning %', ftime(); RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION validate_constraints() RETURNS VOID AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I.%I VALIDATE CONSTRAINT %I;', current_database(), nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated IS FALSE AND nsp.nspname = 'pgstac' LOOP EXECUTE q; END LOOP; END; $$ LANGUAGE PLPGSQL; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value->>'geometry' IS NOT NULL THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value->>'bbox' IS NOT NULL THEN ST_MakeEnvelope( (value->'bbox'->>0)::float, (value->'bbox'->>1)::float, (value->'bbox'->>2)::float, (value->'bbox'->>3)::float, 4326 ) ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'start_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT COALESCE( (value->'properties'->>'datetime')::timestamptz, (value->'properties'->>'end_datetime')::timestamptz ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_daterange(value jsonb) RETURNS tstzrange AS $$ SELECT tstzrange(stac_datetime(value),stac_end_datetime(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS collections ( id VARCHAR GENERATED ALWAYS AS (content->>'id') STORED PRIMARY KEY, content JSONB ); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection_id text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, properties jsonb NOT NULL, content JSONB NOT NULL ) PARTITION BY RANGE (datetime) ; CREATE OR REPLACE FUNCTION properties_idx (IN content jsonb) RETURNS jsonb AS $$ with recursive extract_all as ( select ARRAY[key]::text[] as path, ARRAY[key]::text[] as fullpath, value FROM jsonb_each(content->'properties') union all select CASE WHEN obj_key IS NOT NULL THEN path || obj_key ELSE path END, path || coalesce(obj_key, (arr_key- 1)::text), coalesce(obj_value, arr_value) from extract_all left join lateral jsonb_each(case jsonb_typeof(value) when 'object' then value end) as o(obj_key, obj_value) on jsonb_typeof(value) = 'object' left join lateral jsonb_array_elements(case jsonb_typeof(value) when 'array' then value end) with ordinality as a(arr_value, arr_key) on jsonb_typeof(value) = 'array' where obj_key is not null or arr_key is not null ) , paths AS ( select array_to_string(path, '.') as path, value FROM extract_all WHERE jsonb_typeof(value) NOT IN ('array','object') ), grouped AS ( SELECT path, jsonb_agg(distinct value) vals FROM paths group by path ) SELECT coalesce(jsonb_object_agg(path, CASE WHEN jsonb_array_length(vals)=1 THEN vals->0 ELSE vals END) - '{datetime}'::text[], '{}'::jsonb) FROM grouped ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET JIT TO OFF; CREATE INDEX "datetime_idx" ON items (datetime); CREATE INDEX "end_datetime_idx" ON items (end_datetime); CREATE INDEX "properties_idx" ON items USING GIN (properties jsonb_path_ops); CREATE INDEX "collection_idx" ON items (collection_id); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE UNIQUE INDEX "items_id_datetime_idx" ON items (datetime, id); ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection_id) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION analyze_empty_partitions() RETURNS VOID AS $$ DECLARE p text; BEGIN FOR p IN SELECT partition FROM all_items_partitions WHERE est_cnt = 0 LOOP EXECUTE format('ANALYZE %I;', p); END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION items_partition_name(timestamptz) RETURNS text AS $$ SELECT to_char($1, '"items_p"IYYY"w"IW'); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION items_partition_exists(text) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=$1); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_exists(timestamptz) RETURNS boolean AS $$ SELECT EXISTS (SELECT 1 FROM pg_catalog.pg_class WHERE relname=items_partition_name($1)); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION items_partition_create_worker(partition text, partition_start timestamptz, partition_end timestamptz) RETURNS VOID AS $$ DECLARE err_context text; BEGIN EXECUTE format( $f$ CREATE TABLE IF NOT EXISTS %1$I PARTITION OF items FOR VALUES FROM (%2$L) TO (%3$L); CREATE UNIQUE INDEX IF NOT EXISTS %4$I ON %1$I (id); $f$, partition, partition_start, partition_end, concat(partition, '_id_pk') ); EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH to pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(ts timestamptz) RETURNS text AS $$ DECLARE partition text := items_partition_name(ts); partition_start timestamptz; partition_end timestamptz; BEGIN IF items_partition_exists(partition) THEN RETURN partition; END IF; partition_start := date_trunc('week', ts); partition_end := partition_start + '1 week'::interval; PERFORM items_partition_create_worker(partition, partition_start, partition_end); RAISE NOTICE 'partition: %', partition; RETURN partition; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_partition_create(st timestamptz, et timestamptz) RETURNS SETOF text AS $$ WITH t AS ( SELECT generate_series( date_trunc('week',st), date_trunc('week', et), '1 week'::interval ) w ) SELECT items_partition_create(w) FROM t; $$ LANGUAGE SQL; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_ignore_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT DO NOTHING ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_ignore_insert_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_ignore_insert_triggerfunc(); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_upsert_insert_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; BEGIN CREATE TEMP TABLE new_partitions ON COMMIT DROP AS SELECT items_partition_name(stac_datetime(content)) as partition, date_trunc('week', min(stac_datetime(content))) as partition_start FROM newdata GROUP BY 1; -- set statslastupdated in cache to be old enough cache always regenerated SELECT array_agg(partition) INTO _partitions FROM new_partitions; UPDATE search_wheres SET statslastupdated = NULL WHERE _where IN ( SELECT _where FROM search_wheres sw WHERE sw.partitions && _partitions FOR UPDATE SKIP LOCKED ) ; FOR p IN SELECT new_partitions.partition, new_partitions.partition_start, new_partitions.partition_start + '1 week'::interval as partition_end FROM new_partitions LOOP RAISE NOTICE 'Getting partition % ready.', p.partition; IF NOT items_partition_exists(p.partition) THEN RAISE NOTICE 'Creating partition %.', p.partition; PERFORM items_partition_create_worker(p.partition, p.partition_start, p.partition_end); END IF; PERFORM drop_partition_constraints(p.partition); END LOOP; INSERT INTO items (id, geometry, collection_id, datetime, end_datetime, properties, content) SELECT content->>'id', stac_geom(content), content->>'collection', stac_datetime(content), stac_end_datetime(content), properties_idx(content), content FROM newdata ON CONFLICT (datetime, id) DO UPDATE SET content = EXCLUDED.content WHERE items.content IS DISTINCT FROM EXCLUDED.content ; DELETE FROM items_staging; FOR p IN SELECT new_partitions.partition FROM new_partitions LOOP RAISE NOTICE 'Setting constraints for partition %.', p.partition; PERFORM partition_checks(p.partition); END LOOP; DROP TABLE IF EXISTS new_partitions; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_upsert_insert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_upsert_insert_triggerfunc(); CREATE OR REPLACE FUNCTION items_update_triggerfunc() RETURNS TRIGGER AS $$ DECLARE BEGIN NEW.id := NEW.content->>'id'; NEW.datetime := stac_datetime(NEW.content); NEW.end_datetime := stac_end_datetime(NEW.content); NEW.collection_id := NEW.content->>'collection'; NEW.geometry := stac_geom(NEW.content); NEW.properties := properties_idx(NEW.content); IF TG_OP = 'UPDATE' AND NEW IS NOT DISTINCT FROM OLD THEN RETURN NULL; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_update_trigger BEFORE UPDATE ON items FOR EACH ROW EXECUTE PROCEDURE items_update_triggerfunc(); /* View to get a table of available items partitions with date ranges */ DROP VIEW IF EXISTS all_items_partitions CASCADE; CREATE VIEW all_items_partitions AS WITH base AS (SELECT c.oid::pg_catalog.regclass::text as partition, pg_catalog.pg_get_expr(c.relpartbound, c.oid) as _constraint, regexp_matches( pg_catalog.pg_get_expr(c.relpartbound, c.oid), E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) as t, reltuples::bigint as est_cnt FROM pg_catalog.pg_class c, pg_catalog.pg_inherits i WHERE c.oid = i.inhrelid AND i.inhparent = 'items'::regclass) SELECT partition, tstzrange( t[1]::timestamptz, t[2]::timestamptz ), t[1]::timestamptz as pstart, t[2]::timestamptz as pend, est_cnt FROM base ORDER BY 2 desc; CREATE OR REPLACE VIEW items_partitions AS SELECT * FROM all_items_partitions WHERE est_cnt>0; CREATE OR REPLACE FUNCTION item_by_id(_id text) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_item(_id text) RETURNS jsonb AS $$ SELECT content FROM items WHERE id=_id; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_item(_id text) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(data jsonb) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN UPDATE items SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection_id=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION items_path( IN dotpath text, OUT field text, OUT path text, OUT path_txt text, OUT jsonpath text, OUT eq text ) RETURNS RECORD AS $$ DECLARE path_elements text[]; last_element text; BEGIN dotpath := replace(trim(dotpath), 'properties.', ''); IF dotpath = '' THEN RETURN; END IF; path_elements := string_to_array(dotpath, '.'); jsonpath := NULL; IF path_elements[1] IN ('id','geometry','datetime') THEN field := path_elements[1]; path_elements := path_elements[2:]; ELSIF path_elements[1] = 'collection' THEN field := 'collection_id'; path_elements := path_elements[2:]; ELSIF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN field := 'content'; ELSE field := 'content'; path_elements := '{properties}'::text[] || path_elements; END IF; IF cardinality(path_elements)<1 THEN path := field; path_txt := field; jsonpath := '$'; eq := NULL; RETURN; END IF; last_element := path_elements[cardinality(path_elements)]; path_elements := path_elements[1:cardinality(path_elements)-1]; jsonpath := concat(array_to_string('{$}'::text[] || array_map_ident(path_elements), '.'), '.', quote_ident(last_element)); path_elements := array_map_literal(path_elements); path := format($F$ properties->%s $F$, quote_literal(dotpath)); path_txt := format($F$ properties->>%s $F$, quote_literal(dotpath)); eq := format($F$ properties @? '$.%s[*] ? (@ == %%s) '$F$, quote_ident(dotpath)); RAISE NOTICE 'ITEMS PATH -- % % % % %', field, path, path_txt, jsonpath, eq; RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION parse_dtrange(IN _indate jsonb, OUT _tstzrange tstzrange) AS $$ WITH t AS ( SELECT CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp', 'infinity'] WHEN _indate ? 'interval' THEN textarr(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN textarr(_indate) ELSE regexp_split_to_array( btrim(_indate::text,'"'), '/' ) END AS arr ) , t1 AS ( SELECT CASE WHEN array_upper(arr,1) = 1 OR arr[1] = '..' OR arr[1] IS NULL THEN '-infinity'::timestamptz ELSE arr[1]::timestamptz END AS st, CASE WHEN array_upper(arr,1) = 1 THEN arr[1]::timestamptz WHEN arr[2] = '..' OR arr[2] IS NULL THEN 'infinity'::timestamptz ELSE arr[2]::timestamptz END AS et FROM t ) SELECT tstzrange(st,et) FROM t1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION cql_and_append(existing jsonb, newfilters jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN existing ? 'filter' AND newfilters IS NOT NULL THEN jsonb_build_object( 'and', jsonb_build_array( existing->'filter', newfilters ) ) ELSE newfilters END; $$ LANGUAGE SQL; -- ADDs base filters (ids, collections, datetime, bbox, intersects) that are -- added outside of the filter/query in the stac request CREATE OR REPLACE FUNCTION add_filters_to_cql(j jsonb) RETURNS jsonb AS $$ DECLARE newprop jsonb; newprops jsonb := '[]'::jsonb; BEGIN IF j ? 'ids' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"id"}'::jsonb, j->'ids' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'collections' THEN newprop := jsonb_build_object( 'in', jsonb_build_array( '{"property":"collection"}'::jsonb, j->'collections' ) ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'datetime' THEN newprop := format( '{"anyinteracts":[{"property":"datetime"}, %s]}', j->'datetime' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'bbox' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'bbox' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; IF j ? 'intersects' THEN newprop := format( '{"intersects":[{"property":"geometry"}, %s]}', j->'intersects' ); newprops := jsonb_insert(newprops, '{1}', newprop); END IF; RAISE NOTICE 'newprops: %', newprops; IF newprops IS NOT NULL AND jsonb_array_length(newprops) > 0 THEN return jsonb_set( j, '{filter}', cql_and_append(j, jsonb_build_object('and', newprops)) ) - '{ids,collections,datetime,bbox,intersects}'::text[]; END IF; return j; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cqlfilter(j jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(j->'query') ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ), t3 AS ( SELECT jsonb_strip_nulls(jsonb_build_object( 'and', jsonb_agg( jsonb_build_object( key, jsonb_build_array( jsonb_build_object('property',property), value ) ) ) )) as qcql FROM t2 ) SELECT CASE WHEN qcql IS NOT NULL THEN jsonb_set(j, '{filter}', cql_and_append(j, qcql)) - 'query' ELSE j END FROM t3 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_stac_query(j jsonb) RETURNS text AS $$ DECLARE _where text := ''; dtrange tstzrange; geom geometry; BEGIN IF j ? 'ids' THEN _where := format('%s AND id = ANY (%L) ', _where, textarr(j->'ids')); END IF; IF j ? 'collections' THEN _where := format('%s AND collection_id = ANY (%L) ', _where, textarr(j->'collections')); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); _where := format('%s AND datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', _where, upper(dtrange), lower(dtrange) ); END IF; IF j ? 'bbox' THEN geom := bbox_geom(j->'bbox'); _where := format('%s AND geometry && %L', _where, geom ); END IF; IF j ? 'intersects' THEN geom := st_geomfromgeojson(j->>'intersects'); _where := format('%s AND st_intersects(geometry, %L)', _where, geom ); END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ CREATE OR REPLACE FUNCTION cql_query_op(j jsonb, _op text DEFAULT NULL) RETURNS text AS $$ DECLARE jtype text := jsonb_typeof(j); op text := lower(_op); ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN %s AND %s", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; args text[] := NULL; BEGIN RAISE NOTICE 'j: %, op: %, jtype: %', j, op, jtype; -- for in, convert value, list to array syntax to match other ops IF op = 'in' and j ? 'value' and j ? 'list' THEN j := jsonb_build_array( j->'value', j->'list'); jtype := 'array'; RAISE NOTICE 'IN: j: %, jtype: %', j, jtype; END IF; IF op = 'between' and j ? 'value' and j ? 'lower' and j ? 'upper' THEN j := jsonb_build_array( j->'value', j->'lower', j->'upper'); jtype := 'array'; RAISE NOTICE 'BETWEEN: j: %, jtype: %', j, jtype; END IF; IF op = 'not' AND jtype = 'object' THEN j := jsonb_build_array( j ); jtype := 'array'; RAISE NOTICE 'NOT: j: %, jtype: %', j, jtype; END IF; -- Set Lower Case on Both Arguments When Case Insensitive Flag Set IF op in ('eq','lt','lte','gt','gte','like') AND jsonb_typeof(j->2) = 'boolean' THEN IF (j->>2)::boolean THEN RETURN format(concat('(',ops->>op,')'), cql_query_op(jsonb_build_array(j->0), 'lower'), cql_query_op(jsonb_build_array(j->1), 'lower')); END IF; END IF; -- Special Case when comparing a property in a jsonb field to a string or number using eq -- Allows to leverage GIN index on jsonb fields IF op = 'eq' THEN IF j->0 ? 'property' AND jsonb_typeof(j->1) IN ('number','string') AND (items_path(j->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(j->0->>'property')).eq, j->1); END IF; END IF; IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, j); END IF; IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, j); END IF; IF jtype = 'object' THEN RAISE NOTICE 'parsing object'; IF j ? 'property' THEN -- Convert the property to be used as an identifier return (items_path(j->>'property')).path_txt; ELSIF _op IS NULL THEN -- Iterate to convert elements in an object where the operator has not been set -- Combining with AND SELECT array_to_string(array_agg(cql_query_op(e.value, e.key)), ' AND ') INTO ret FROM jsonb_each(j) e; RETURN ret; END IF; END IF; IF jtype = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jtype ='number' THEN RETURN (j->>0)::numeric; END IF; IF jtype = 'array' AND op IS NULL THEN RAISE NOTICE 'Parsing array into array arg. j: %', j; SELECT format($f$ '{%s}'::text[] $f$, string_agg(e,',')) INTO ret FROM jsonb_array_elements_text(j) e; RETURN ret; END IF; -- If the type of the passed json is an array -- Calculate the arguments that will be passed to functions/operators IF jtype = 'array' THEN RAISE NOTICE 'Parsing array into args. j: %', j; -- If any argument is numeric, cast any text arguments to numeric IF j @? '$[*] ? (@.type() == "number")' THEN SELECT INTO args array_agg(concat('(',cql_query_op(e),')::numeric')) FROM jsonb_array_elements(j) e; ELSE SELECT INTO args array_agg(cql_query_op(e)) FROM jsonb_array_elements(j) e; END IF; --RETURN args; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; IF op IS NULL THEN RETURN args::text[]; END IF; IF args IS NULL OR cardinality(args) < 1 THEN RAISE NOTICE 'No Args'; RETURN ''; END IF; IF op IN ('and','or') THEN SELECT CONCAT( '(', array_to_string(args, UPPER(CONCAT(' ',op,' '))), ')' ) INTO ret FROM jsonb_array_elements(j) e; RETURN ret; END IF; -- If the op is in the ops json then run using the template in the json IF ops ? op THEN RAISE NOTICE 'ARGS: % MAPPED: %',args, array_map_literal(args); RETURN format(concat('(',ops->>op,')'), VARIADIC args); END IF; RETURN j->>0; END; $$ LANGUAGE PLPGSQL; /* cql_query_op -- Parses a CQL query operation, recursing when necessary IN jsonb -- a subelement from a valid stac query IN text -- the operator being used on elements passed in RETURNS a SQL fragment to be used in a WHERE clause */ DROP FUNCTION IF EXISTS cql2_query; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, recursion int DEFAULT 0) RETURNS text AS $$ DECLARE args jsonb := j->'args'; jtype text := jsonb_typeof(j->'args'); op text := lower(j->>'op'); arg jsonb; argtext text; argstext text[] := '{}'::text[]; inobj jsonb; _numeric text := ''; ops jsonb := '{ "eq": "%s = %s", "lt": "%s < %s", "lte": "%s <= %s", "gt": "%s > %s", "gte": "%s >= %s", "le": "%s <= %s", "ge": "%s >= %s", "=": "%s = %s", "<": "%s < %s", "<=": "%s <= %s", ">": "%s > %s", ">=": "%s >= %s", "like": "%s LIKE %s", "ilike": "%s ILIKE %s", "+": "%s + %s", "-": "%s - %s", "*": "%s * %s", "/": "%s / %s", "in": "%s = ANY (%s)", "not": "NOT (%s)", "between": "%s BETWEEN (%2$s)[1] AND (%2$s)[2]", "lower":" lower(%s)", "upper":" upper(%s)", "isnull": "%s IS NULL" }'::jsonb; ret text; BEGIN RAISE NOTICE 'j: %s', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RAISE NOTICE 'upper %s',jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ; RETURN cql2_query( jsonb_build_object( 'op', 'upper', 'args', jsonb_build_array( j-> 'upper') ) ); END IF; IF j ? 'lower' THEN RETURN cql2_query( jsonb_build_object( 'op', 'lower', 'args', jsonb_build_array( j-> 'lower') ) ); END IF; IF j ? 'args' AND jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; -- END Cases where no further nesting is expected IF j ? 'op' THEN -- Special case to use JSONB index for equality IF op IN ('eq', '=') AND args->0 ? 'property' AND jsonb_typeof(args->1) IN ('number', 'string') AND (items_path(args->0->>'property')).eq IS NOT NULL THEN RETURN format((items_path(args->0->>'property')).eq, args->1); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; -- In Query - separate into separate eq statements so that we can use eq jsonb optimization IF op = 'in' THEN RAISE NOTICE '% IN args: %', repeat(' ', recursion), args; SELECT INTO inobj jsonb_agg( jsonb_build_object( 'op', 'eq', 'args', jsonb_build_array( args->0 , v) ) ) FROM jsonb_array_elements( args->1) v; RETURN cql2_query(jsonb_build_object('op','or','args',inobj)); END IF; END IF; IF j ? 'property' THEN RETURN (items_path(j->>'property')).path_txt; END IF; IF j ? 'timestamp' THEN RETURN quote_literal(j->>'timestamp'); END IF; RAISE NOTICE '%jtype: %',repeat(' ', recursion), jtype; IF jsonb_typeof(j) = 'number' THEN RETURN format('%L::numeric', j->>0); END IF; IF jsonb_typeof(j) = 'string' THEN RETURN quote_literal(j->>0); END IF; IF jsonb_typeof(j) = 'array' THEN IF j @? '$[*] ? (@.type() == "number")' THEN RETURN CONCAT(quote_literal(textarr(j)::text), '::numeric[]'); ELSE RETURN CONCAT(quote_literal(textarr(j)::text), '::text[]'); END IF; END IF; RAISE NOTICE 'ARGS after array cleaning: %', args; RAISE NOTICE '%beforeargs op: %, args: %',repeat(' ', recursion), op, args; IF j ? 'args' THEN FOR arg in SELECT * FROM jsonb_array_elements(args) LOOP argtext := cql2_query(arg, recursion + 1); RAISE NOTICE '% -- arg: %, argtext: %', repeat(' ', recursion), arg, argtext; argstext := argstext || argtext; END LOOP; END IF; RAISE NOTICE '%afterargs op: %, argstext: %',repeat(' ', recursion), op, argstext; IF op IN ('and', 'or') THEN RAISE NOTICE 'inand op: %, argstext: %', op, argstext; SELECT concat(' ( ',array_to_string(array_agg(e), concat(' ',op,' ')),' ) ') INTO ret FROM unnest(argstext) e; RETURN ret; END IF; IF ops ? op THEN IF argstext[2] ~* 'numeric' THEN argstext := ARRAY[concat('(',argstext[1],')::numeric')] || argstext[2:3]; END IF; RETURN format(concat('(',ops->>op,')'), VARIADIC argstext); END IF; RAISE NOTICE '%op: %, argstext: %',repeat(' ', recursion), op, argstext; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION cql_to_where(_search jsonb = '{}'::jsonb) RETURNS text AS $$ DECLARE filterlang text; search jsonb := _search; base_where text; _where text; BEGIN RAISE NOTICE 'SEARCH CQL Final: %', search; filterlang := COALESCE( search->>'filter-lang', get_setting('default-filter-lang', _search->'conf') ); base_where := base_stac_query(search); IF filterlang = 'cql-json' THEN search := query_to_cqlfilter(search); -- search := add_filters_to_cql(search); _where := cql_query_op(search->'filter'); ELSE _where := cql2_query(search->'filter'); END IF; IF trim(_where) = '' THEN _where := NULL; END IF; _where := coalesce(_where, ' TRUE '); RETURN format('( %s ) %s ', _where, base_where); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION field_orderby(p text) RETURNS text AS $$ WITH t AS ( SELECT replace(trim(substring(indexdef from 'btree \((.*)\)')),' ','')as s FROM pg_indexes WHERE schemaname='pgstac' AND tablename='items' AND indexdef ~* 'btree' AND indexdef ~* 'properties' ) SELECT s FROM t WHERE strpos(s, lower(trim(p)))>0; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce(field_orderby((items_path(value->>'field')).path_txt), (items_path(value->>'field')).path) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM item_by_id(token_id) as items; END IF; RAISE NOTICE 'TOKEN ID: %', token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (items_path(value->>'field')).path, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); BEGIN SELECT * INTO sw FROM search_wheres WHERE _where=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed.'; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', context_stats_ttl(conf), now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', context(conf), sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > context_stats_ttl(conf) OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE _where = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain_json, 'strict $.**."Relation Name" ? (@ != null)' ) j ), ordered AS ( SELECT p FROM t ORDER BY p DESC -- SELECT p FROM t JOIN items_partitions -- ON (t.p = items_partitions.partition) -- ORDER BY pstart DESC ) SELECT array_agg(p) INTO partitions FROM ordered; i := clock_timestamp() - t; RAISE NOTICE 'Time for explain + join: %', clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); sw.partitions := partitions; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF context(conf) = 'on' OR ( context(conf) = 'auto' AND ( sw.estimated_count < context_estimated_count(conf) OR sw.estimated_cost < context_estimated_cost(conf) ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, partitions = sw.partitions, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; CREATE OR REPLACE FUNCTION items_count(_where text) RETURNS bigint AS $$ DECLARE cnt bigint; BEGIN EXECUTE format('SELECT count(*) FROM items WHERE %s', _where) INTO cnt; RETURN cnt; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := cql_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record items%ROWTYPE; last_record items%ROWTYPE; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN FOR id IN SELECT jsonb_array_elements_text(_search->'ids') LOOP INSERT INTO results (content) SELECT content FROM item_by_id(id) WHERE content IS NOT NULL; END LOOP; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := iter_record; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record.content); -- out_records := out_records || last_record.content; ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %. Total Time %', clock_timestamp()-timer, ftime(); timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_record))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); RAISE NOTICE 'token_type: %, has_next: %, has_prev: %', token_type, has_next, has_prev; IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; -- include/exclude any fields following fields extension IF _search ? 'fields' THEN IF _search->'fields' ? 'exclude' THEN excludes=textarr(_search->'fields'->'exclude'); END IF; IF _search->'fields' ? 'include' THEN includes=textarr(_search->'fields'->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; SELECT jsonb_agg(filter_jsonb(row, includes, excludes)) INTO out_records FROM jsonb_array_elements(out_records) row; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET jit TO off ; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb[] := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); IF fields IS NOT NULL THEN IF fields ? 'fields' THEN fields := fields->'fields'; END IF; IF fields ? 'exclude' THEN excludes=textarr(fields->'exclude'); END IF; IF fields ? 'include' THEN includes=textarr(fields->'include'); IF array_length(includes, 1)>0 AND NOT 'id' = ANY (includes) THEN includes = includes || '{id}'; END IF; END IF; END IF; RAISE NOTICE 'fields: %, includes: %, excludes: %', fields, includes, excludes; FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; curs = create_cursor(query); LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; IF fields IS NOT NULL THEN out_records := out_records || filter_jsonb(iter_record.content, includes, excludes); ELSE out_records := out_records || iter_record.content; END IF; IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', array_to_json(out_records)::jsonb ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; SELECT set_version('0.4.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.5.0-0.5.1.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.content_nonhydrated(_item pgstac.items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql STABLE PARALLEL SAFE AS $function$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry)::jsonb; END IF; IF include_field('bbox', fields) THEN bbox := geom_bbox(_item.geometry)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'bbox',bbox, 'collection', _item.collection ) || _item.content; RETURN output; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path TO 'pgstac', 'public' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(content_dehydrate(first_record)))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; SELECT set_version('0.5.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.5.0.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT $1->>0; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props ? 'end_datetime' THEN dt := props->'start_datetime'; edt := props->'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->'datetime'; edt := props->'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE EXCEPTION 'Either datetime or both start_datetime and end_datetime must be set.'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id', 'links', '[]'::jsonb ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id), name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('in', '%s = ANY (%s)', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN (%2$s)[1] AND (%2$s)[2]', NULL), ('isnull', '%s IS NULL', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template; ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN format('upper(%s)', cql2_query(j->'upper')); END IF; IF j ? 'lower' THEN RETURN format('lower(%s)', cql2_query(j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RETURN format( '%s = ANY (%L)', cql2_query(args->0), to_text_array(args->1) ); END IF; IF op = 'between' THEN SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; RETURN format( '%s BETWEEN %s and %s', cql2_query(args->0, wrapper), cql2_query(args->1->0, wrapper), cql2_query(args->1->1, wrapper) ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF j ? 'property' THEN RETURN (queryable(j->>'property')).expression; END IF; IF wrapper IS NOT NULL THEN EXECUTE format('SELECT %I(%L)', wrapper, j) INTO literal; RAISE NOTICE '% % %',wrapper, j, literal; RETURN format('%I(%L)', wrapper, j); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb, _collection jsonb) RETURNS jsonb AS $$ SELECT jsonb_object_agg( key, CASE WHEN jsonb_typeof(c.value) = 'object' AND jsonb_typeof(i.value) = 'object' THEN content_slim(i.value, c.value) ELSE i.value END ) FROM jsonb_each(_item) as i LEFT JOIN jsonb_each(_collection) as c USING (key) WHERE i.value IS DISTINCT FROM c.value ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT content_slim(_item - '{id,type,collection,geometry,bbox}'::text[], collection_base_item(_item->>'collection')); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := coalesce(fields->'includes', fields->'include', '[]'::jsonb); excludes jsonb := coalesce(fields->'excludes', fields->'exclude', '[]'::jsonb); BEGIN IF f IS NULL THEN RETURN NULL; ELSIF jsonb_array_length(includes)>0 AND includes ? f THEN RETURN TRUE; ELSIF jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; ELSIF jsonb_array_length(includes)>0 AND NOT includes ? f THEN RETURN FALSE; END IF; RETURN TRUE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION key_filter(IN k text, IN val jsonb, INOUT kf jsonb, OUT include boolean) AS $$ DECLARE includes jsonb := coalesce(kf->'includes', kf->'include', '[]'::jsonb); excludes jsonb := coalesce(kf->'excludes', kf->'exclude', '[]'::jsonb); BEGIN RAISE NOTICE '% % %', k, val, kf; include := TRUE; IF k = 'properties' AND NOT excludes ? 'properties' THEN excludes := excludes || '["properties"]'; include := TRUE; RAISE NOTICE 'Prop include %', include; ELSIF jsonb_array_length(excludes)>0 AND excludes ? k THEN include := FALSE; ELSIF jsonb_array_length(includes)>0 AND NOT includes ? k THEN include := FALSE; ELSIF jsonb_array_length(includes)>0 AND includes ? k THEN includes := '[]'::jsonb; RAISE NOTICE 'KF: %', kf; END IF; kf := jsonb_build_object('includes', includes, 'excludes', excludes); RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION strip_assets(a jsonb) RETURNS jsonb AS $$ WITH t AS (SELECT * FROM jsonb_each(a)) SELECT jsonb_object_agg(key, value) FROM t WHERE value ? 'href'; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _collection jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT jsonb_strip_nulls(jsonb_object_agg( key, CASE WHEN key = 'properties' AND include_field('properties', fields) THEN i.value WHEN key = 'properties' THEN content_hydrate(i.value, c.value, kf) WHEN c.value IS NULL AND key != 'properties' THEN i.value WHEN key = 'assets' AND jsonb_typeof(c.value) = 'object' AND jsonb_typeof(i.value) = 'object' THEN strip_assets(content_hydrate(i.value, c.value, kf)) WHEN jsonb_typeof(c.value) = 'object' AND jsonb_typeof(i.value) = 'object' THEN content_hydrate(i.value, c.value, kf) ELSE coalesce(i.value, c.value) END )) FROM jsonb_each(coalesce(_item,'{}'::jsonb)) as i FULL JOIN jsonb_each(coalesce(_collection,'{}'::jsonb)) as c USING (key) JOIN LATERAL ( SELECT kf, include FROM key_filter(key, i.value, fields) ) as k ON (include) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry)::jsonb; END IF; IF include_field('bbox', fields) THEN bbox := geom_bbox(_item.geometry)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'bbox',bbox, 'collection', _item.collection ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL STABLE; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN SELECT delete_item(content->>'id', content->>'collection'); SELECT create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT content_hydrate(items, _search->'fields') FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; last_record := content_hydrate(iter_record, _search->'fields'); IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(content_dehydrate(first_record)))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.5.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.5.1-0.6.0.sql ================================================ SET SEARCH_PATH to pgstac, public; alter table "pgstac"."partitions" drop constraint "partitions_collection_fkey"; drop function if exists "pgstac"."content_hydrate"(_item jsonb, _collection jsonb, fields jsonb); drop function if exists "pgstac"."content_slim"(_item jsonb, _collection jsonb); drop function if exists "pgstac"."key_filter"(k text, val jsonb, INOUT kf jsonb, OUT include boolean); drop function if exists "pgstac"."strip_assets"(a jsonb); alter table "pgstac"."partitions" add constraint "partitions_collection_fkey" FOREIGN KEY (collection) REFERENCES pgstac.collections(id) ON DELETE CASCADE; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.content_hydrate(_base_item jsonb, _item jsonb, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $function$ ; CREATE OR REPLACE FUNCTION pgstac.explode_dotpaths(j jsonb) RETURNS SETOF text[] LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $function$ ; CREATE OR REPLACE FUNCTION pgstac.explode_dotpaths_recurse(j jsonb) RETURNS SETOF text[] LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $function$ ; CREATE OR REPLACE FUNCTION pgstac.jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb LANGUAGE plpgsql IMMUTABLE AS $function$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields": []}'::jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE AS $function$ SELECT jsonb_exclude(jsonb_include(j, f), f); $function$ ; CREATE OR REPLACE FUNCTION pgstac.jsonb_include(j jsonb, f jsonb) RETURNS jsonb LANGUAGE plpgsql IMMUTABLE AS $function$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb LANGUAGE plpgsql IMMUTABLE AS $function$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE AS $function$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.partitions_delete_trigger_func() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE AS $function$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_base_item(content jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $function$ ; CREATE OR REPLACE FUNCTION pgstac.collections_trigger_func() RETURNS trigger LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' AS $function$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.content_hydrate(_item pgstac.items, _collection pgstac.collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql STABLE PARALLEL SAFE AS $function$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.content_nonhydrated(_item pgstac.items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql STABLE PARALLEL SAFE AS $function$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.content_slim(_item jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $function$ ; CREATE OR REPLACE FUNCTION pgstac.delete_item(_id text, _collection text DEFAULT NULL::text) RETURNS void LANGUAGE plpgsql AS $function$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $function$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.partition_name(collection text, dt timestamp with time zone, OUT partition_name text, OUT partition_range tstzrange) RETURNS record LANGUAGE plpgsql STABLE AS $function$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path TO 'pgstac', 'public' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.stac_daterange(value jsonb) RETURNS tstzrange LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE SET "TimeZone" TO 'UTC' AS $function$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_item(content jsonb) RETURNS void LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' AS $function$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $function$ ; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON pgstac.partitions FOR EACH ROW EXECUTE FUNCTION pgstac.partitions_delete_trigger_func(); SELECT set_version('0.6.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.5.1.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT $1->>0; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props ? 'end_datetime' THEN dt := props->'start_datetime'; edt := props->'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->'datetime'; edt := props->'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE EXCEPTION 'Either datetime or both start_datetime and end_datetime must be set.'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id', 'links', '[]'::jsonb ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id), name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('in', '%s = ANY (%s)', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN (%2$s)[1] AND (%2$s)[2]', NULL), ('isnull', '%s IS NULL', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template; ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN format('upper(%s)', cql2_query(j->'upper')); END IF; IF j ? 'lower' THEN RETURN format('lower(%s)', cql2_query(j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RETURN format( '%s = ANY (%L)', cql2_query(args->0), to_text_array(args->1) ); END IF; IF op = 'between' THEN SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; RETURN format( '%s BETWEEN %s and %s', cql2_query(args->0, wrapper), cql2_query(args->1->0, wrapper), cql2_query(args->1->1, wrapper) ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF j ? 'property' THEN RETURN (queryable(j->>'property')).expression; END IF; IF wrapper IS NOT NULL THEN EXECUTE format('SELECT %I(%L)', wrapper, j) INTO literal; RAISE NOTICE '% % %',wrapper, j, literal; RETURN format('%I(%L)', wrapper, j); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb, _collection jsonb) RETURNS jsonb AS $$ SELECT jsonb_object_agg( key, CASE WHEN jsonb_typeof(c.value) = 'object' AND jsonb_typeof(i.value) = 'object' THEN content_slim(i.value, c.value) ELSE i.value END ) FROM jsonb_each(_item) as i LEFT JOIN jsonb_each(_collection) as c USING (key) WHERE i.value IS DISTINCT FROM c.value ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT content_slim(_item - '{id,type,collection,geometry,bbox}'::text[], collection_base_item(_item->>'collection')); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := coalesce(fields->'includes', fields->'include', '[]'::jsonb); excludes jsonb := coalesce(fields->'excludes', fields->'exclude', '[]'::jsonb); BEGIN IF f IS NULL THEN RETURN NULL; ELSIF jsonb_array_length(includes)>0 AND includes ? f THEN RETURN TRUE; ELSIF jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; ELSIF jsonb_array_length(includes)>0 AND NOT includes ? f THEN RETURN FALSE; END IF; RETURN TRUE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION key_filter(IN k text, IN val jsonb, INOUT kf jsonb, OUT include boolean) AS $$ DECLARE includes jsonb := coalesce(kf->'includes', kf->'include', '[]'::jsonb); excludes jsonb := coalesce(kf->'excludes', kf->'exclude', '[]'::jsonb); BEGIN RAISE NOTICE '% % %', k, val, kf; include := TRUE; IF k = 'properties' AND NOT excludes ? 'properties' THEN excludes := excludes || '["properties"]'; include := TRUE; RAISE NOTICE 'Prop include %', include; ELSIF jsonb_array_length(excludes)>0 AND excludes ? k THEN include := FALSE; ELSIF jsonb_array_length(includes)>0 AND NOT includes ? k THEN include := FALSE; ELSIF jsonb_array_length(includes)>0 AND includes ? k THEN includes := '[]'::jsonb; RAISE NOTICE 'KF: %', kf; END IF; kf := jsonb_build_object('includes', includes, 'excludes', excludes); RETURN; END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION strip_assets(a jsonb) RETURNS jsonb AS $$ WITH t AS (SELECT * FROM jsonb_each(a)) SELECT jsonb_object_agg(key, value) FROM t WHERE value ? 'href'; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _collection jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT jsonb_strip_nulls(jsonb_object_agg( key, CASE WHEN key = 'properties' AND include_field('properties', fields) THEN i.value WHEN key = 'properties' THEN content_hydrate(i.value, c.value, kf) WHEN c.value IS NULL AND key != 'properties' THEN i.value WHEN key = 'assets' AND jsonb_typeof(c.value) = 'object' AND jsonb_typeof(i.value) = 'object' THEN strip_assets(content_hydrate(i.value, c.value, kf)) WHEN jsonb_typeof(c.value) = 'object' AND jsonb_typeof(i.value) = 'object' THEN content_hydrate(i.value, c.value, kf) ELSE coalesce(i.value, c.value) END )) FROM jsonb_each(coalesce(_item,'{}'::jsonb)) as i FULL JOIN jsonb_each(coalesce(_collection,'{}'::jsonb)) as c USING (key) JOIN LATERAL ( SELECT kf, include FROM key_filter(key, i.value, fields) ) as k ON (include) ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry)::jsonb; END IF; IF include_field('bbox', fields) THEN bbox := geom_bbox(_item.geometry)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'bbox',bbox, 'collection', _item.collection ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry)::jsonb; END IF; IF include_field('bbox', fields) THEN bbox := geom_bbox(_item.geometry)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'bbox',bbox, 'collection', _item.collection ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL STABLE; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN SELECT delete_item(content->>'id', content->>'collection'); SELECT create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; IF cntr = 1 THEN first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(content_dehydrate(first_record)))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.5.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.0-0.6.1.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.to_text(jsonb) RETURNS text LANGUAGE sql IMMUTABLE STRICT AS $function$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $function$ ; SELECT set_version('0.6.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.0.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT $1->>0; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('in', '%s = ANY (%s)', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN (%2$s)[1] AND (%2$s)[2]', NULL), ('isnull', '%s IS NULL', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template; ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN format('upper(%s)', cql2_query(j->'upper')); END IF; IF j ? 'lower' THEN RETURN format('lower(%s)', cql2_query(j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RETURN format( '%s = ANY (%L)', cql2_query(args->0), to_text_array(args->1) ); END IF; IF op = 'between' THEN SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; RETURN format( '%s BETWEEN %s and %s', cql2_query(args->0, wrapper), cql2_query(args->1->0, wrapper), cql2_query(args->1->1, wrapper) ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF j ? 'property' THEN RETURN (queryable(j->>'property')).expression; END IF; IF wrapper IS NOT NULL THEN EXECUTE format('SELECT %I(%L)', wrapper, j) INTO literal; RAISE NOTICE '% % %',wrapper, j, literal; RETURN format('%I(%L)', wrapper, j); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _base_item jsonb, _item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.1-0.6.2.sql ================================================ SET SEARCH_PATH to pgstac, public; SELECT set_version('0.6.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.1.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('in', '%s = ANY (%s)', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN (%2$s)[1] AND (%2$s)[2]', NULL), ('isnull', '%s IS NULL', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template; ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN format('upper(%s)', cql2_query(j->'upper')); END IF; IF j ? 'lower' THEN RETURN format('lower(%s)', cql2_query(j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RETURN format( '%s = ANY (%L)', cql2_query(args->0), to_text_array(args->1) ); END IF; IF op = 'between' THEN SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; RETURN format( '%s BETWEEN %s and %s', cql2_query(args->0, wrapper), cql2_query(args->1->0, wrapper), cql2_query(args->1->1, wrapper) ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF j ? 'property' THEN RETURN (queryable(j->>'property')).expression; END IF; IF wrapper IS NOT NULL THEN EXECUTE format('SELECT %I(%L)', wrapper, j) INTO literal; RAISE NOTICE '% % %',wrapper, j, literal; RETURN format('%I(%L)', wrapper, j); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _base_item jsonb, _item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.10-0.6.11.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; UPDATE pgstac_settings SET name='default_filter_lang' WHERE name='default-filter-lang'; CREATE OR REPLACE FUNCTION pgstac.stac_search_to_where(j jsonb) RETURNS text LANGUAGE plpgsql STABLE AS $function$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $function$ ; SELECT set_version('0.6.11'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.10.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', 'https://example.org/queryables', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition ) ) FROM queryables WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR collection_ids IS NULL OR _collection_ids && collection_ids ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample int DEFAULT 5) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize bigint; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; IF _tablesample * .01 * psize < 10 THEN _tablesample := 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows', _tablesample, _collection, _partition, psize; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) WHERE q.definition IS NULL ) SELECT %L, key, jsonb_build_object('type',jsonb_typeof(value)) as definition, CASE jsonb_typeof(value) WHEN 'number' THEN 'to_float' WHEN 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample int DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.10'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.11-0.6.12.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; alter table "pgstac"."queryables" drop constraint "queryables_name_key"; drop function if exists "pgstac"."missing_queryables"(_collection text, _tablesample integer); drop function if exists "pgstac"."missing_queryables"(_tablesample integer); alter table "pgstac"."stac_extensions" drop constraint "stac_extensions_pkey"; drop index if exists "pgstac"."queryables_name_key"; drop index if exists "pgstac"."stac_extensions_pkey"; alter table "pgstac"."stac_extensions" drop column "enableable"; alter table "pgstac"."stac_extensions" drop column "enbabled_by_default"; alter table "pgstac"."stac_extensions" drop column "name"; alter table "pgstac"."stac_extensions" add column "content" jsonb; alter table "pgstac"."stac_extensions" alter column "url" set not null; CREATE UNIQUE INDEX stac_extensions_pkey ON pgstac.stac_extensions USING btree (url); alter table "pgstac"."stac_extensions" add constraint "stac_extensions_pkey" PRIMARY KEY using index "stac_extensions_pkey"; set check_function_bodies = off; CREATE OR REPLACE PROCEDURE pgstac.analyze_items() LANGUAGE plpgsql AS $procedure$ DECLARE q text; BEGIN FOR q IN SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LOOP RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $procedure$ ; CREATE OR REPLACE FUNCTION pgstac.check_pgstac_settings(_sysmem text DEFAULT NULL::text) RETURNS void LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' SET client_min_messages TO 'notice' AS $function$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT COALESCE($1,$2); $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_queryables() RETURNS jsonb LANGUAGE sql AS $function$ SELECT get_queryables(NULL::text[]); $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_token_val_str(_field text, _item pgstac.items) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE literal text; BEGIN RAISE NOTICE '% %', _field, _item; CREATE TEMP TABLE _token_item ON COMMIT DROP AS SELECT (_item).*; EXECUTE format($q$ SELECT quote_literal(%s) FROM _token_item $q$, _field) INTO literal; DROP TABLE IF EXISTS _token_item; RETURN literal; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.jsonb_array_unique(j jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE AS $function$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $function$ ; CREATE OR REPLACE FUNCTION pgstac.jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE AS $function$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $function$ ; CREATE OR REPLACE FUNCTION pgstac.jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE AS $function$ SELECT nullif_jsonbnullempty(greatest(a, b)); $function$ ; CREATE OR REPLACE FUNCTION pgstac.jsonb_least(a jsonb, b jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE AS $function$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $function$ ; CREATE OR REPLACE FUNCTION pgstac.missing_queryables(_collection text, _tablesample double precision DEFAULT 5, minrows double precision DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) LANGUAGE plpgsql AS $function$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.missing_queryables(_tablesample double precision DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) LANGUAGE sql AS $function$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.nullif_jsonbnullempty(j jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE STRICT AS $function$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $function$ ; CREATE OR REPLACE FUNCTION pgstac.schema_qualify_refs(url text, j jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE STRICT AS $function$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $function$ ; create or replace view "pgstac"."stac_extension_queryables" as SELECT DISTINCT j.key AS name, pgstac.schema_qualify_refs(e.url, j.value) AS definition FROM pgstac.stac_extensions e, LATERAL jsonb_each((((e.content -> 'definitions'::text) -> 'fields'::text) -> 'properties'::text)) j(key, value); CREATE OR REPLACE PROCEDURE pgstac.validate_constraints() LANGUAGE plpgsql AS $procedure$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $procedure$ ; CREATE OR REPLACE FUNCTION pgstac.get_queryables(_collection_ids text[] DEFAULT NULL::text[]) RETURNS jsonb LANGUAGE plpgsql STABLE AS $function$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_token_filter(_search jsonb DEFAULT '{}'::jsonb, token_rec jsonb DEFAULT NULL::jsonb) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; token_item items%ROWTYPE; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; token_item := jsonb_populate_record(null::items, token_rec); RAISE NOTICE 'TOKEN ITEM ----- %', token_item; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=get_token_val_str(_field, token_item); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN IF sort._val IS NULL THEN orfilters := orfilters || format('(%s IS NOT NULL)', sort._field); ELSE orfilters := orfilters || format('(%s %s %s)', sort._field, CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; ELSE IF sort._val IS NULL THEN orfilters := orfilters || format('(%s AND %s IS NOT NULL)', array_to_string(andfilters, ' AND '), sort._field); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), sort._field, CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; END IF; IF sort._val IS NULL THEN andfilters := andfilters || format('%s IS NULL', sort._field ); ELSE andfilters := andfilters || format('%s = %s', sort._field, sort._val ); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $function$ ; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); SELECT set_version('0.6.12'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.11.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', 'https://example.org/queryables', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition ) ) FROM queryables WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR collection_ids IS NULL OR _collection_ids && collection_ids ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample int DEFAULT 5) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize bigint; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; IF _tablesample * .01 * psize < 10 THEN _tablesample := 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows', _tablesample, _collection, _partition, psize; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) WHERE q.definition IS NULL ) SELECT %L, key, jsonb_build_object('type',jsonb_typeof(value)) as definition, CASE jsonb_typeof(value) WHEN 'number' THEN 'to_float' WHEN 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample int DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.11'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.12-0.6.13.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path TO 'pgstac', 'public' SET cursor_tuple_fraction TO '1' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results (content) SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) SELECT jsonb_agg(content) INTO out_records FROM ordered; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; SELECT set_version('0.6.13'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.12.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; BEGIN FOR q IN SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LOOP RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE literal text; BEGIN RAISE NOTICE '% %', _field, _item; CREATE TEMP TABLE _token_item ON COMMIT DROP AS SELECT (_item).*; EXECUTE format($q$ SELECT quote_literal(%s) FROM _token_item $q$, _field) INTO literal; DROP TABLE IF EXISTS _token_item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; token_item items%ROWTYPE; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; token_item := jsonb_populate_record(null::items, token_rec); RAISE NOTICE 'TOKEN ITEM ----- %', token_item; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=get_token_val_str(_field, token_item); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN IF sort._val IS NULL THEN orfilters := orfilters || format('(%s IS NOT NULL)', sort._field); ELSE orfilters := orfilters || format('(%s %s %s)', sort._field, CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; ELSE IF sort._val IS NULL THEN orfilters := orfilters || format('(%s AND %s IS NOT NULL)', array_to_string(andfilters, ' AND '), sort._field); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), sort._field, CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; END IF; IF sort._val IS NULL THEN andfilters := andfilters || format('%s IS NULL', sort._field ); ELSE andfilters := andfilters || format('%s = %s', sort._field, sort._val ); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.12'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.13-0.7.0.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL drop trigger if exists "queryables_collection_trigger" on "pgstac"."collections"; drop trigger if exists "partitions_delete_trigger" on "pgstac"."partitions"; drop trigger if exists "partitions_trigger" on "pgstac"."partitions"; alter table "pgstac"."partitions" drop constraint "partitions_collection_fkey"; alter table "pgstac"."partitions" drop constraint "prange"; drop function if exists "pgstac"."create_queryable_indexes"(); drop function if exists "pgstac"."partition_collection"(collection text, strategy pgstac.partition_trunc_strategy); drop function if exists "pgstac"."partitions_delete_trigger_func"(); drop function if exists "pgstac"."partitions_trigger_func"(); drop function if exists "pgstac"."parse_dtrange"(_indate jsonb, relative_base timestamp with time zone); drop view if exists "pgstac"."partition_steps"; alter table "pgstac"."partitions" drop constraint "partitions_pkey"; drop index if exists "pgstac"."partitions_pkey"; select 1; -- drop index if exists "pgstac"."prange"; drop index if exists "pgstac"."partitions_range_idx"; drop table "pgstac"."partitions"; create table "pgstac"."partition_stats" ( "partition" text not null, "dtrange" tstzrange, "edtrange" tstzrange, "spatial" geometry, "last_updated" timestamp with time zone, "keys" text[] ); create table "pgstac"."query_queue" ( "query" text not null, "added" timestamp with time zone default now() ); create table "pgstac"."query_queue_history" ( "query" text, "added" timestamp with time zone not null, "finished" timestamp with time zone not null default now(), "error" text ); alter table "pgstac"."collections" alter column "partition_trunc" set data type text using "partition_trunc"::text; drop type "pgstac"."partition_trunc_strategy"; CREATE UNIQUE INDEX partition_stats_pkey ON pgstac.partition_stats USING btree (partition); CREATE UNIQUE INDEX query_queue_pkey ON pgstac.query_queue USING btree (query); CREATE INDEX partitions_range_idx ON pgstac.partition_stats USING gist (dtrange); alter table "pgstac"."partition_stats" add constraint "partition_stats_pkey" PRIMARY KEY using index "partition_stats_pkey"; alter table "pgstac"."query_queue" add constraint "query_queue_pkey" PRIMARY KEY using index "query_queue_pkey"; alter table "pgstac"."collections" add constraint "collections_partition_trunc_check" CHECK ((partition_trunc = ANY (ARRAY['year'::text, 'month'::text]))) not valid; alter table "pgstac"."collections" validate constraint "collections_partition_trunc_check"; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.check_partition(_collection text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); RETURN _partition_name; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_extent(_collection text, runupdate boolean DEFAULT false) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.constraint_tstzrange(expr text) RETURNS tstzrange LANGUAGE sql IMMUTABLE PARALLEL SAFE STRICT AS $function$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; $q$, t, format('%s_dt', t), t, format('%s_dt', t) ); ELSE q :=format( $q$ ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; $q$, t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t) ); END IF; PERFORM run_or_queue(q); RETURN t; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.drop_table_constraints(t text) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS record LANGUAGE plpgsql STABLE STRICT AS $function$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_setting_bool(_setting text, conf jsonb DEFAULT NULL::jsonb) RETURNS boolean LANGUAGE sql AS $function$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $function$ ; CREATE OR REPLACE FUNCTION pgstac.maintain_partition_queries(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false) RETURNS SETOF text LANGUAGE plpgsql AS $function$ DECLARE parent text; level int; isleaf bool; collection collections%ROWTYPE; subpart text; baseidx text; queryable_name text; queryable_property_index_type text; queryable_property_wrapper text; queryable_parsed RECORD; deletedidx pg_indexes%ROWTYPE; q text; idx text; collection_partition bigint; _concurrently text := ''; BEGIN RAISE NOTICE 'Maintaining partition: %', part; IF get_setting_bool('use_queue') THEN _concurrently='CONCURRENTLY'; END IF; -- Get root partition SELECT parentrelid::text, pt.isleaf, pt.level INTO parent, isleaf, level FROM pg_partition_tree('items') pt WHERE relid::text = part; IF NOT FOUND THEN RAISE NOTICE 'Partition % Does Not Exist In Partition Tree', part; RETURN; END IF; -- If this is a parent partition, recurse to leaves IF NOT isleaf THEN FOR subpart IN SELECT relid::text FROM pg_partition_tree(part) WHERE relid::text != part LOOP RAISE NOTICE 'Recursing to %', subpart; RETURN QUERY SELECT * FROM maintain_partition_queries(subpart, dropindexes, rebuildindexes); END LOOP; RETURN; -- Don't continue since not an end leaf END IF; -- Get collection collection_partition := ((regexp_match(part, E'^_items_([0-9]+)'))[1])::bigint; RAISE NOTICE 'COLLECTION PARTITION: %', collection_partition; SELECT * INTO STRICT collection FROM collections WHERE key = collection_partition; RAISE NOTICE 'COLLECTION ID: %s', collection.id; -- Create temp table with existing indexes CREATE TEMP TABLE existing_indexes ON COMMIT DROP AS SELECT * FROM pg_indexes WHERE schemaname='pgstac' AND tablename=part; -- Check if index exists for each queryable. FOR queryable_name, queryable_property_index_type, queryable_property_wrapper IN SELECT name, COALESCE(property_index_type, 'BTREE'), COALESCE(property_wrapper, 'to_text') FROM queryables WHERE name NOT in ('id', 'datetime', 'geometry') AND ( collection_ids IS NULL OR collection_ids = '{}'::text[] OR collection.id = ANY (collection_ids) ) UNION ALL SELECT 'datetime desc, end_datetime', 'BTREE', '' UNION ALL SELECT 'geometry', 'GIST', '' UNION ALL SELECT 'id', 'BTREE', '' LOOP baseidx := format( $q$ ON %I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, part, queryable_property_index_type, queryable_property_wrapper, queryable_name ); RAISE NOTICE 'BASEIDX: %', baseidx; RAISE NOTICE 'IDXSEARCH: %', format($q$[(']%s[')]$q$, queryable_name); -- If index already exists, delete it from existing indexes type table FOR deletedidx IN DELETE FROM existing_indexes WHERE indexdef ~* format($q$[(']%s[')]$q$, queryable_name) RETURNING * LOOP RAISE NOTICE 'EXISTING INDEX: %', deletedidx; IF NOT FOUND THEN -- index did not exist, create it RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); ELSIF rebuildindexes THEN RETURN NEXT format('REINDEX %I %s;', deletedidx.indexname, _concurrently); END IF; END LOOP; END LOOP; -- Remove indexes that were not expected FOR idx IN SELECT indexname::text FROM existing_indexes LOOP RAISE WARNING 'Index: % is not defined by queryables.', idx; IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', idx); END IF; END LOOP; DROP TABLE existing_indexes; RAISE NOTICE 'Returning from maintain_partition_queries.'; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.maintain_partitions(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false) RETURNS void LANGUAGE sql AS $function$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $function$ ; CREATE OR REPLACE FUNCTION pgstac.partition_after_triggerfunc() RETURNS trigger LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $function$ ; create or replace view "pgstac"."partition_sys_meta" as SELECT (pg_partition_tree.relid)::text AS partition, replace(replace( CASE WHEN (pg_partition_tree.level = 1) THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, pg_partition_tree.level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_edtrange FROM (((pg_partition_tree('pgstac.items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) LEFT JOIN pg_constraint edt ON (((edt.conrelid = c.oid) AND (edt.contype = 'c'::"char")))) WHERE pg_partition_tree.isleaf; create or replace view "pgstac"."partitions" as SELECT partition_sys_meta.partition, partition_sys_meta.collection, partition_sys_meta.level, partition_sys_meta.reltuples, partition_sys_meta.relhastriggers, partition_sys_meta.partition_dtrange, partition_sys_meta.constraint_dtrange, partition_sys_meta.constraint_edtrange, partition_stats.dtrange, partition_stats.edtrange, partition_stats.spatial, partition_stats.last_updated, partition_stats.keys FROM (pgstac.partition_sys_meta LEFT JOIN pgstac.partition_stats USING (partition)); create or replace view "pgstac"."pgstac_indexes" as SELECT i.schemaname, i.tablename, i.indexname, i.indexdef, COALESCE((regexp_match(i.indexdef, '\(([a-zA-Z]+)\)'::text))[1], (regexp_match(i.indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'::text))[1], CASE WHEN (i.indexdef ~* '\(datetime desc, end_datetime\)'::text) THEN 'datetime_end_datetime'::text ELSE NULL::text END) AS field, pg_table_size(((i.indexname)::text)::regclass) AS index_size, pg_size_pretty(pg_table_size(((i.indexname)::text)::regclass)) AS index_size_pretty FROM pg_indexes i WHERE ((i.schemaname = 'pgstac'::name) AND (i.tablename ~ '_items_'::text)); create or replace view "pgstac"."pgstac_indexes_stats" as SELECT i.schemaname, i.tablename, i.indexname, i.indexdef, COALESCE((regexp_match(i.indexdef, '\(([a-zA-Z]+)\)'::text))[1], (regexp_match(i.indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'::text))[1], CASE WHEN (i.indexdef ~* '\(datetime desc, end_datetime\)'::text) THEN 'datetime_end_datetime'::text ELSE NULL::text END) AS field, pg_table_size(((i.indexname)::text)::regclass) AS index_size, pg_size_pretty(pg_table_size(((i.indexname)::text)::regclass)) AS index_size_pretty, s.n_distinct, ((s.most_common_vals)::text)::text[] AS most_common_vals, ((s.most_common_freqs)::text)::text[] AS most_common_freqs, ((s.histogram_bounds)::text)::text[] AS histogram_bounds, s.correlation FROM (pg_indexes i LEFT JOIN pg_stats s ON ((s.tablename = i.indexname))) WHERE ((i.schemaname = 'pgstac'::name) AND (i.tablename ~ '_items_'::text)); CREATE OR REPLACE FUNCTION pgstac.queue_timeout() RETURNS interval LANGUAGE sql AS $function$ SELECT set_config( 'statement_timeout', t2s(coalesce( get_setting('queue_timeout'), '1h' )), false )::interval; $function$ ; CREATE OR REPLACE FUNCTION pgstac.repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT false) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.run_or_queue(query text) RETURNS void LANGUAGE plpgsql AS $function$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $function$ ; CREATE OR REPLACE PROCEDURE pgstac.run_queued_queries() LANGUAGE plpgsql AS $procedure$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $procedure$ ; CREATE OR REPLACE FUNCTION pgstac.run_queued_queries_intransaction() RETURNS integer LANGUAGE plpgsql AS $function$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.t2s(text) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE STRICT AS $function$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_partition_stats(_partition text, istrigger boolean DEFAULT false) RETURNS void LANGUAGE plpgsql STRICT AS $function$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, partitions.collection INTO cdtrange, cedtrange, collection FROM partitions WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_partition_stats_q(_partition text, istrigger boolean DEFAULT false) RETURNS void LANGUAGE plpgsql AS $function$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.all_collections() RETURNS jsonb LANGUAGE sql SET search_path TO 'pgstac', 'public' AS $function$ SELECT jsonb_agg(content) FROM collections; $function$ ; CREATE OR REPLACE PROCEDURE pgstac.analyze_items() LANGUAGE plpgsql AS $procedure$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP RAISE NOTICE '% % %', clock_timestamp(), timeout_ts, current_setting('statement_timeout', TRUE); SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; RAISE NOTICE '%', queue_timeout(); END LOOP; END; $procedure$ ; CREATE OR REPLACE FUNCTION pgstac.check_pgstac_settings(_sysmem text DEFAULT NULL::text) RETURNS void LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' SET client_min_messages TO 'notice' AS $function$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collections_trigger_func() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_item(_id text, _collection text DEFAULT NULL::text) RETURNS jsonb LANGUAGE sql STABLE SET search_path TO 'pgstac', 'public' AS $function$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_setting(_setting text, conf jsonb DEFAULT NULL::jsonb) RETURNS text LANGUAGE sql AS $function$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $function$ ; CREATE OR REPLACE FUNCTION pgstac.item_by_id(_id text, _collection text DEFAULT NULL::text) RETURNS pgstac.items LANGUAGE plpgsql STABLE SET search_path TO 'pgstac', 'public' AS $function$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.parse_dtrange(_indate jsonb, relative_base timestamp with time zone DEFAULT date_trunc('hour'::text, CURRENT_TIMESTAMP)) RETURNS tstzrange LANGUAGE plpgsql STABLE PARALLEL SAFE STRICT SET "TimeZone" TO 'UTC' AS $function$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $function$ ; create or replace view "pgstac"."partition_steps" as SELECT partitions.partition AS name, date_trunc('month'::text, lower(partitions.partition_dtrange)) AS sdate, (date_trunc('month'::text, upper(partitions.partition_dtrange)) + '1 mon'::interval) AS edate FROM pgstac.partitions WHERE ((partitions.partition_dtrange IS NOT NULL) AND (partitions.partition_dtrange <> 'empty'::tstzrange)) ORDER BY partitions.dtrange; CREATE OR REPLACE FUNCTION pgstac.queryables_trigger_func() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' SET cursor_tuple_fraction TO '1' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results (content) SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) SELECT jsonb_agg(content) INTO out_records FROM ordered; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) RETURNS pgstac.searches LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $function$ ; CREATE OR REPLACE PROCEDURE pgstac.validate_constraints() LANGUAGE plpgsql AS $procedure$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $procedure$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) RETURNS pgstac.search_wheres LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $function$ ; CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON pgstac.items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION pgstac.partition_after_triggerfunc(); CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON pgstac.items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION pgstac.partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON pgstac.items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION pgstac.partition_after_triggerfunc(); -- END migra calculated SQL INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'true') ON CONFLICT DO NOTHING ; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.13-0.7.3.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL drop trigger if exists "queryables_collection_trigger" on "pgstac"."collections"; drop trigger if exists "partitions_delete_trigger" on "pgstac"."partitions"; drop trigger if exists "partitions_trigger" on "pgstac"."partitions"; alter table "pgstac"."partitions" drop constraint "partitions_collection_fkey"; alter table "pgstac"."partitions" drop constraint "prange"; drop function if exists "pgstac"."create_queryable_indexes"(); drop function if exists "pgstac"."partition_collection"(collection text, strategy pgstac.partition_trunc_strategy); drop function if exists "pgstac"."partitions_delete_trigger_func"(); drop function if exists "pgstac"."partitions_trigger_func"(); drop view if exists "pgstac"."partition_steps"; alter table "pgstac"."partitions" drop constraint "partitions_pkey"; drop index if exists "pgstac"."partitions_pkey"; select 1; -- drop index if exists "pgstac"."prange"; drop index if exists "pgstac"."partitions_range_idx"; drop table "pgstac"."partitions"; create table "pgstac"."partition_stats" ( "partition" text not null, "dtrange" tstzrange, "edtrange" tstzrange, "spatial" geometry, "last_updated" timestamp with time zone, "keys" text[] ); create table "pgstac"."query_queue" ( "query" text not null, "added" timestamp with time zone default now() ); create table "pgstac"."query_queue_history" ( "query" text, "added" timestamp with time zone not null, "finished" timestamp with time zone not null default now(), "error" text ); alter table "pgstac"."collections" alter column "partition_trunc" set data type text using "partition_trunc"::text; drop type "pgstac"."partition_trunc_strategy"; CREATE UNIQUE INDEX partition_stats_pkey ON pgstac.partition_stats USING btree (partition); CREATE UNIQUE INDEX query_queue_pkey ON pgstac.query_queue USING btree (query); CREATE INDEX queryables_collection_idx ON pgstac.queryables USING gin (collection_ids); CREATE INDEX partitions_range_idx ON pgstac.partition_stats USING gist (dtrange); alter table "pgstac"."partition_stats" add constraint "partition_stats_pkey" PRIMARY KEY using index "partition_stats_pkey"; alter table "pgstac"."query_queue" add constraint "query_queue_pkey" PRIMARY KEY using index "query_queue_pkey"; alter table "pgstac"."collections" add constraint "collections_partition_trunc_check" CHECK ((partition_trunc = ANY (ARRAY['year'::text, 'month'::text]))) not valid; alter table "pgstac"."collections" validate constraint "collections_partition_trunc_check"; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.check_partition(_collection text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); RETURN _partition_name; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_extent(_collection text, runupdate boolean DEFAULT false) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.constraint_tstzrange(expr text) RETURNS tstzrange LANGUAGE sql IMMUTABLE PARALLEL SAFE STRICT AS $function$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.drop_table_constraints(t text) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS record LANGUAGE plpgsql STABLE STRICT AS $function$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_setting_bool(_setting text, conf jsonb DEFAULT NULL::jsonb) RETURNS boolean LANGUAGE sql AS $function$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $function$ ; CREATE OR REPLACE FUNCTION pgstac.indexdef(q pgstac.queryables) RETURNS text LANGUAGE plpgsql IMMUTABLE AS $function$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.maintain_partition_queries(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false, idxconcurrently boolean DEFAULT false) RETURNS SETOF text LANGUAGE plpgsql AS $function$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.maintain_partitions(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false) RETURNS void LANGUAGE sql AS $function$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $function$ ; CREATE OR REPLACE FUNCTION pgstac.normalize_indexdef(def text) RETURNS text LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE STRICT AS $function$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.partition_after_triggerfunc() RETURNS trigger LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $function$ ; create or replace view "pgstac"."partition_sys_meta" as SELECT (pg_partition_tree.relid)::text AS partition, replace(replace( CASE WHEN (pg_partition_tree.level = 1) THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, pg_partition_tree.level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_edtrange FROM (((pg_partition_tree('pgstac.items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) LEFT JOIN pg_constraint edt ON (((edt.conrelid = c.oid) AND (edt.contype = 'c'::"char")))) WHERE pg_partition_tree.isleaf; create or replace view "pgstac"."partitions" as SELECT partition_sys_meta.partition, partition_sys_meta.collection, partition_sys_meta.level, partition_sys_meta.reltuples, partition_sys_meta.relhastriggers, partition_sys_meta.partition_dtrange, partition_sys_meta.constraint_dtrange, partition_sys_meta.constraint_edtrange, partition_stats.dtrange, partition_stats.edtrange, partition_stats.spatial, partition_stats.last_updated, partition_stats.keys FROM (pgstac.partition_sys_meta LEFT JOIN pgstac.partition_stats USING (partition)); create or replace view "pgstac"."pgstac_indexes" as SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(i.indexdef, (i.indexname)::text, ''::text), 'pgstac.'::text, ''::text), ' \t\n'::text), '[ ]+'::text, ' '::text, 'g'::text) AS idx, COALESCE((regexp_match(i.indexdef, '\(([a-zA-Z]+)\)'::text))[1], (regexp_match(i.indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'::text))[1], CASE WHEN (i.indexdef ~* '\(datetime desc, end_datetime\)'::text) THEN 'datetime'::text ELSE NULL::text END) AS field, pg_table_size(((i.indexname)::text)::regclass) AS index_size, pg_size_pretty(pg_table_size(((i.indexname)::text)::regclass)) AS index_size_pretty FROM pg_indexes i WHERE ((i.schemaname = 'pgstac'::name) AND (i.tablename ~ '_items_'::text) AND (i.indexdef !~* ' only '::text)); create or replace view "pgstac"."pgstac_indexes_stats" as SELECT i.schemaname, i.tablename, i.indexname, i.indexdef, COALESCE((regexp_match(i.indexdef, '\(([a-zA-Z]+)\)'::text))[1], (regexp_match(i.indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'::text))[1], CASE WHEN (i.indexdef ~* '\(datetime desc, end_datetime\)'::text) THEN 'datetime_end_datetime'::text ELSE NULL::text END) AS field, pg_table_size(((i.indexname)::text)::regclass) AS index_size, pg_size_pretty(pg_table_size(((i.indexname)::text)::regclass)) AS index_size_pretty, s.n_distinct, ((s.most_common_vals)::text)::text[] AS most_common_vals, ((s.most_common_freqs)::text)::text[] AS most_common_freqs, ((s.histogram_bounds)::text)::text[] AS histogram_bounds, s.correlation FROM (pg_indexes i LEFT JOIN pg_stats s ON ((s.tablename = i.indexname))) WHERE ((i.schemaname = 'pgstac'::name) AND (i.tablename ~ '_items_'::text)); CREATE OR REPLACE FUNCTION pgstac.queryable_signature(n text, c text[]) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT concat(n, c); $function$ ; CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables'; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM collections LEFT JOIN unnest(NEW.collection_ids) c ON (collections.id = c) WHERE c IS NULL ) THEN RAISE foreign_key_violation; RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation; RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.queue_timeout() RETURNS interval LANGUAGE sql AS $function$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $function$ ; CREATE OR REPLACE FUNCTION pgstac.repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT false) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.run_or_queue(query text) RETURNS void LANGUAGE plpgsql AS $function$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $function$ ; CREATE OR REPLACE PROCEDURE pgstac.run_queued_queries() LANGUAGE plpgsql AS $procedure$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $procedure$ ; CREATE OR REPLACE FUNCTION pgstac.run_queued_queries_intransaction() RETURNS integer LANGUAGE plpgsql AS $function$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.t2s(text) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE STRICT AS $function$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $function$ ; CREATE OR REPLACE FUNCTION pgstac.unnest_collection(collection_ids text[] DEFAULT NULL::text[]) RETURNS SETOF text LANGUAGE plpgsql STABLE AS $function$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_partition_stats(_partition text, istrigger boolean DEFAULT false) RETURNS void LANGUAGE plpgsql STRICT AS $function$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, partitions.collection INTO cdtrange, cedtrange, collection FROM partitions WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_partition_stats_q(_partition text, istrigger boolean DEFAULT false) RETURNS void LANGUAGE plpgsql AS $function$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.all_collections() RETURNS jsonb LANGUAGE sql SET search_path TO 'pgstac', 'public' AS $function$ SELECT jsonb_agg(content) FROM collections; $function$ ; CREATE OR REPLACE PROCEDURE pgstac.analyze_items() LANGUAGE plpgsql AS $procedure$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $procedure$ ; CREATE OR REPLACE FUNCTION pgstac.check_pgstac_settings(_sysmem text DEFAULT NULL::text) RETURNS void LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' SET client_min_messages TO 'notice' AS $function$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collections_trigger_func() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_item(_id text, _collection text DEFAULT NULL::text) RETURNS jsonb LANGUAGE sql STABLE SET search_path TO 'pgstac', 'public' AS $function$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_setting(_setting text, conf jsonb DEFAULT NULL::jsonb) RETURNS text LANGUAGE sql AS $function$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_token_filter(_search jsonb DEFAULT '{}'::jsonb, token_rec jsonb DEFAULT NULL::jsonb) RETURNS text LANGUAGE plpgsql SET transform_null_equals TO 'true' AS $function$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; token_item items%ROWTYPE; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); IF token_id IS NULL OR token_id = '' THEN RAISE WARNING 'next or prev set, but no token id found'; RETURN NULL; END IF; SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; token_item := jsonb_populate_record(null::items, token_rec); RAISE NOTICE 'TOKEN ITEM ----- %', token_item; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=get_token_val_str(_field, token_item); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s < %s) OR (%s IS NULL) )$f$, sort._field, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; --orfilter := format('%s IS NULL', sort._field); ELSE orfilter := format($f$( (%s > %s) OR (%s IS NULL) )$f$, sort._field, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.item_by_id(_id text, _collection text DEFAULT NULL::text) RETURNS pgstac.items LANGUAGE plpgsql STABLE SET search_path TO 'pgstac', 'public' AS $function$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.items_staging_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.parse_dtrange(_indate jsonb, relative_base timestamp with time zone DEFAULT date_trunc('hour'::text, CURRENT_TIMESTAMP)) RETURNS tstzrange LANGUAGE plpgsql STABLE PARALLEL SAFE STRICT SET "TimeZone" TO 'UTC' AS $function$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $function$ ; create or replace view "pgstac"."partition_steps" as SELECT partitions.partition AS name, date_trunc('month'::text, lower(partitions.partition_dtrange)) AS sdate, (date_trunc('month'::text, upper(partitions.partition_dtrange)) + '1 mon'::interval) AS edate FROM pgstac.partitions WHERE ((partitions.partition_dtrange IS NOT NULL) AND (partitions.partition_dtrange <> 'empty'::tstzrange)) ORDER BY partitions.dtrange; CREATE OR REPLACE FUNCTION pgstac.queryables_trigger_func() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' SET cursor_tuple_fraction TO '1' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) SELECT jsonb_agg(content) INTO out_records FROM ordered; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) RETURNS pgstac.searches LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $function$ ; CREATE OR REPLACE PROCEDURE pgstac.validate_constraints() LANGUAGE plpgsql AS $procedure$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $procedure$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) RETURNS pgstac.search_wheres LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $function$ ; CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON pgstac.items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION pgstac.partition_after_triggerfunc(); CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON pgstac.items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION pgstac.partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON pgstac.items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION pgstac.partition_after_triggerfunc(); CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON pgstac.queryables FOR EACH ROW EXECUTE FUNCTION pgstac.queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON pgstac.queryables FOR EACH ROW WHEN (((new.name = old.name) AND (new.collection_ids IS DISTINCT FROM old.collection_ids))) EXECUTE FUNCTION pgstac.queryables_constraint_triggerfunc(); -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false') ON CONFLICT DO NOTHING ; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.13.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; BEGIN FOR q IN SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LOOP RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE literal text; BEGIN RAISE NOTICE '% %', _field, _item; CREATE TEMP TABLE _token_item ON COMMIT DROP AS SELECT (_item).*; EXECUTE format($q$ SELECT quote_literal(%s) FROM _token_item $q$, _field) INTO literal; DROP TABLE IF EXISTS _token_item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; token_item items%ROWTYPE; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; token_item := jsonb_populate_record(null::items, token_rec); RAISE NOTICE 'TOKEN ITEM ----- %', token_item; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=get_token_val_str(_field, token_item); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN IF sort._val IS NULL THEN orfilters := orfilters || format('(%s IS NOT NULL)', sort._field); ELSE orfilters := orfilters || format('(%s %s %s)', sort._field, CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; ELSE IF sort._val IS NULL THEN orfilters := orfilters || format('(%s AND %s IS NOT NULL)', array_to_string(andfilters, ' AND '), sort._field); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), sort._field, CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; END IF; IF sort._val IS NULL THEN andfilters := andfilters || format('%s IS NULL', sort._field ); ELSE andfilters := andfilters || format('%s = %s', sort._field, sort._val ); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results (content) SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) SELECT jsonb_agg(content) INTO out_records FROM ordered; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.13'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.2-0.6.3.sql ================================================ SET SEARCH_PATH to pgstac, public; drop function if exists "pgstac"."content_hydrate"(_base_item jsonb, _item jsonb, fields jsonb); set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.content_hydrate(_item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $function$ ; SELECT set_version('0.6.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.2.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('in', '%s = ANY (%s)', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN (%2$s)[1] AND (%2$s)[2]', NULL), ('isnull', '%s IS NULL', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template; ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN format('upper(%s)', cql2_query(j->'upper')); END IF; IF j ? 'lower' THEN RETURN format('lower(%s)', cql2_query(j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RETURN format( '%s = ANY (%L)', cql2_query(args->0), to_text_array(args->1) ); END IF; IF op = 'between' THEN SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; RETURN format( '%s BETWEEN %s and %s', cql2_query(args->0, wrapper), cql2_query(args->1->0, wrapper), cql2_query(args->1->1, wrapper) ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF j ? 'property' THEN RETURN (queryable(j->>'property')).expression; END IF; IF wrapper IS NOT NULL THEN EXECUTE format('SELECT %I(%L)', wrapper, j) INTO literal; RAISE NOTICE '% % %',wrapper, j, literal; RETURN format('%I(%L)', wrapper, j); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _base_item jsonb, _item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.3-0.6.4.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('in', '%s = ANY (%s)', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, wrapper text DEFAULT NULL::text) RETURNS text LANGUAGE plpgsql STABLE AS $function$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RETURN format( '%s = ANY (%L)', cql2_query(args->0), to_text_array(args->1) ); END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; SELECT property_wrapper INTO wrapper FROM queryables WHERE name=(arg->>'property') LIMIT 1; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $function$ ; SELECT set_version('0.6.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.3.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('in', '%s = ANY (%s)', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN (%2$s)[1] AND (%2$s)[2]', NULL), ('isnull', '%s IS NULL', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template; ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN format('upper(%s)', cql2_query(j->'upper')); END IF; IF j ? 'lower' THEN RETURN format('lower(%s)', cql2_query(j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RETURN format( '%s = ANY (%L)', cql2_query(args->0), to_text_array(args->1) ); END IF; IF op = 'between' THEN SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; RETURN format( '%s BETWEEN %s and %s', cql2_query(args->0, wrapper), cql2_query(args->1->0, wrapper), cql2_query(args->1->1, wrapper) ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; SELECT (queryable(a->>'property')).wrapper INTO wrapper FROM jsonb_array_elements(args) a WHERE a ? 'property' LIMIT 1; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF j ? 'property' THEN RETURN (queryable(j->>'property')).expression; END IF; IF wrapper IS NOT NULL THEN EXECUTE format('SELECT %I(%L)', wrapper, j) INTO literal; RAISE NOTICE '% % %',wrapper, j, literal; RETURN format('%I(%L)', wrapper, j); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.4-0.6.5.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, wrapper text DEFAULT NULL::text) RETURNS text LANGUAGE plpgsql STABLE AS $function$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; -- RETURN format( -- '%s = ANY (%L)', -- cql2_query(args->0), -- to_text_array(args->1) -- ); END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; SELECT property_wrapper INTO wrapper FROM queryables WHERE name=(arg->>'property') LIMIT 1; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $function$ ; SELECT set_version('0.6.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.4.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('in', '%s = ANY (%s)', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RETURN format( '%s = ANY (%L)', cql2_query(args->0), to_text_array(args->1) ); END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; SELECT property_wrapper INTO wrapper FROM queryables WHERE name=(arg->>'property') LIMIT 1; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.5-0.6.6.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, wrapper text DEFAULT NULL::text) RETURNS text LANGUAGE plpgsql STABLE AS $function$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; SELECT property_wrapper INTO wrapper FROM queryables WHERE name=(arg->>'property') LIMIT 1; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $function$ ; TRUNCATE cql2_ops; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; SELECT set_version('0.6.6'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.5.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; -- RETURN format( -- '%s = ANY (%L)', -- cql2_query(args->0), -- to_text_array(args->1) -- ); END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; SELECT property_wrapper INTO wrapper FROM queryables WHERE name=(arg->>'property') LIMIT 1; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.6-0.6.7.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; SELECT set_version('0.6.7'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.6.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; SELECT property_wrapper INTO wrapper FROM queryables WHERE name=(arg->>'property') LIMIT 1; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.6'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.7-0.6.8.sql ================================================ SET SEARCH_PATH to pgstac, public; drop function if exists "pgstac"."queryable"(dotpath text, OUT path text, OUT expression text, OUT wrapper text); set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.get_queryables(_collection text DEFAULT NULL::text) RETURNS jsonb LANGUAGE sql AS $function$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_queryables(_collection_ids text[] DEFAULT NULL::text[]) RETURNS jsonb LANGUAGE sql STABLE AS $function$ SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', 'https://example.org/queryables', 'type', 'object', 'title', 'Stac Queryables.', 'properties', jsonb_object_agg( name, definition ) ) FROM queryables WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR collection_ids IS NULL OR _collection_ids && collection_ids ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.missing_queryables(_collection text, _tablesample integer DEFAULT 5) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) LANGUAGE plpgsql AS $function$ DECLARE q text; _partition text; explain_json json; psize bigint; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; IF _tablesample * .01 * psize < 10 THEN _tablesample := 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows', _tablesample, _collection, _partition, psize; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) WHERE q.definition IS NULL ) SELECT %L, key, jsonb_build_object('type',jsonb_typeof(value)) as definition, CASE jsonb_typeof(value) WHEN 'number' THEN 'to_float' WHEN 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.missing_queryables(_tablesample integer DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) LANGUAGE sql AS $function$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.queryable(dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text) RETURNS record LANGUAGE plpgsql STABLE STRICT AS $function$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, wrapper text DEFAULT NULL::text) RETURNS text LANGUAGE plpgsql STABLE AS $function$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $function$ ; SELECT set_version('0.6.8'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.7.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; ELSE wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; SELECT property_wrapper INTO wrapper FROM queryables WHERE name=(arg->>'property') LIMIT 1; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.7'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.8-0.6.9.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER SET search_path TO 'pgstac', 'public' SET cursor_tuple_fraction TO '1' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; SELECT set_version('0.6.9'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.8.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', 'https://example.org/queryables', 'type', 'object', 'title', 'Stac Queryables.', 'properties', jsonb_object_agg( name, definition ) ) FROM queryables WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR collection_ids IS NULL OR _collection_ids && collection_ids ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample int DEFAULT 5) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize bigint; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; IF _tablesample * .01 * psize < 10 THEN _tablesample := 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows', _tablesample, _collection, _partition, psize; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) WHERE q.definition IS NULL ) SELECT %L, key, jsonb_build_object('type',jsonb_typeof(value)) as definition, CASE jsonb_typeof(value) WHEN 'number' THEN 'to_float' WHEN 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample int DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.8'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.9-0.6.10.sql ================================================ SET SEARCH_PATH to pgstac, public; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.get_queryables(_collection_ids text[] DEFAULT NULL::text[]) RETURNS jsonb LANGUAGE plpgsql STABLE AS $function$ BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', 'https://example.org/queryables', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition ) ) FROM queryables WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR collection_ids IS NULL OR _collection_ids && collection_ids ); ELSE RETURN NULL; END IF; END; $function$ ; SELECT set_version('0.6.10'); ================================================ FILE: src/pgstac/migrations/pgstac.0.6.9.sql ================================================ CREATE EXTENSION IF NOT EXISTS postgis; CREATE EXTENSION IF NOT EXISTS btree_gist; DO $$ BEGIN CREATE ROLE pgstac_admin; CREATE ROLE pgstac_read; CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default-filter-lang', 'cql2-json'), ('additional_properties', 'true') ON CONFLICT DO NOTHING ; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting) ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( name text PRIMARY KEY, url text, enbabled_by_default boolean NOT NULL DEFAULT TRUE, enableable boolean NOT NULL DEFAULT TRUE ); INSERT INTO stac_extensions (name, url) VALUES ('fields', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#fields'), ('sort','https://api.stacspec.org/v1.0.0-beta.5/item-search#sort'), ('context','https://api.stacspec.org/v1.0.0-beta.5/item-search#context'), ('filter', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#filter'), ('query', 'https://api.stacspec.org/v1.0.0-beta.5/item-search#query') ON CONFLICT (name) DO UPDATE SET url=EXCLUDED.url; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TYPE partition_trunc_strategy AS ENUM ('year', 'month'); CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc partition_trunc_strategy ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; SELECT relid::text INTO partition_name FROM pg_partition_tree('items') WHERE relid::text = partition_name; IF FOUND THEN partition_exists := true; partition_empty := table_empty(partition_name); ELSE partition_exists := false; partition_empty := true; partition_name := format('_items_%s', NEW.key); END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_empty THEN q := format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name ); EXECUTE q; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc AND partition_exists AND NOT partition_empty THEN q := format($q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; $q$, partition_name, partition_name ); EXECUTE q; loadtemp := TRUE; partition_empty := TRUE; partition_exists := FALSE; END IF; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS NOT DISTINCT FROM OLD.partition_trunc THEN RETURN NEW; END IF; IF NEW.partition_trunc IS NULL AND partition_empty THEN RAISE NOTICE '% % % %', partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, NEW.id, concat(partition_name,'_id_idx'), partition_name ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; INSERT INTO partitions (collection, name) VALUES (NEW.id, partition_name); ELSIF partition_empty THEN q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); $q$, partition_name, NEW.id ); RAISE NOTICE 'q: %', q; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; ALTER TABLE partitions DISABLE TRIGGER partitions_delete_trigger; DELETE FROM partitions WHERE collection=NEW.id AND name=partition_name; ALTER TABLE partitions ENABLE TRIGGER partitions_delete_trigger; ELSE RAISE EXCEPTION 'Cannot modify partition % unless empty', partition_name; END IF; IF loadtemp THEN RAISE NOTICE 'Moving data into new partitions.'; q := format($q$ WITH p AS ( SELECT collection, datetime as datetime, end_datetime as end_datetime, (partition_name( collection, datetime )).partition_name as name FROM changepartitionstaging ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; INSERT INTO %I SELECT * FROM changepartitionstaging; DROP TABLE IF EXISTS changepartitionstaging; $q$, partition_name ); EXECUTE q; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION partition_collection(collection text, strategy partition_trunc_strategy) RETURNS text AS $$ UPDATE collections SET partition_trunc=strategy WHERE id=collection RETURNING partition_trunc; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS partitions ( collection text REFERENCES collections(id) ON DELETE CASCADE, name text PRIMARY KEY, partition_range tstzrange NOT NULL DEFAULT tstzrange('-infinity'::timestamptz,'infinity'::timestamptz, '[]'), datetime_range tstzrange, end_datetime_range tstzrange, CONSTRAINT prange EXCLUDE USING GIST ( collection WITH =, partition_range WITH && ) ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partitions USING GIST(partition_range); CREATE OR REPLACE FUNCTION partitions_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; BEGIN RAISE NOTICE 'Partition Delete Trigger. %', OLD.name; EXECUTE format($q$ DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.name ); RAISE NOTICE 'Dropped partition.'; RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_delete_trigger BEFORE DELETE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_delete_trigger_func(); CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange ) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION partitions_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; cq text; parent_name text; partition_trunc text; partition_name text := NEW.name; partition_exists boolean := false; partition_empty boolean := true; partition_range tstzrange; datetime_range tstzrange; end_datetime_range tstzrange; err_context text; mindt timestamptz := lower(NEW.datetime_range); maxdt timestamptz := upper(NEW.datetime_range); minedt timestamptz := lower(NEW.end_datetime_range); maxedt timestamptz := upper(NEW.end_datetime_range); t_mindt timestamptz; t_maxdt timestamptz; t_minedt timestamptz; t_maxedt timestamptz; BEGIN RAISE NOTICE 'Partitions Trigger. %', NEW; datetime_range := NEW.datetime_range; end_datetime_range := NEW.end_datetime_range; SELECT format('_items_%s', key), c.partition_trunc::text INTO parent_name, partition_trunc FROM pgstac.collections c WHERE c.id = NEW.collection; SELECT (pgstac.partition_name(NEW.collection, mindt)).* INTO partition_name, partition_range; NEW.name := partition_name; IF partition_range IS NULL OR partition_range = 'empty'::tstzrange THEN partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; NEW.partition_range := partition_range; IF TG_OP = 'UPDATE' THEN mindt := least(mindt, lower(OLD.datetime_range)); maxdt := greatest(maxdt, upper(OLD.datetime_range)); minedt := least(minedt, lower(OLD.end_datetime_range)); maxedt := greatest(maxedt, upper(OLD.end_datetime_range)); NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); END IF; IF TG_OP = 'INSERT' THEN IF partition_range != tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]') THEN RAISE NOTICE '% % %', partition_name, parent_name, partition_range; q := format($q$ CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, partition_name, parent_name, lower(partition_range), upper(partition_range), format('%s_pkey', partition_name), partition_name ); BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; END IF; END IF; -- Update constraints EXECUTE format($q$ SELECT min(datetime), max(datetime), min(end_datetime), max(end_datetime) FROM %I; $q$, partition_name) INTO t_mindt, t_maxdt, t_minedt, t_maxedt; mindt := least(mindt, t_mindt); maxdt := greatest(maxdt, t_maxdt); minedt := least(mindt, minedt, t_minedt); maxedt := greatest(maxdt, maxedt, t_maxedt); mindt := date_trunc(coalesce(partition_trunc, 'year'), mindt); maxdt := date_trunc(coalesce(partition_trunc, 'year'), maxdt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; minedt := date_trunc(coalesce(partition_trunc, 'year'), minedt); maxedt := date_trunc(coalesce(partition_trunc, 'year'), maxedt - '1 second'::interval) + concat('1 ',coalesce(partition_trunc, 'year'))::interval; IF mindt IS NOT NULL AND maxdt IS NOT NULL AND minedt IS NOT NULL AND maxedt IS NOT NULL THEN NEW.datetime_range := tstzrange(mindt, maxdt, '[]'); NEW.end_datetime_range := tstzrange(minedt, maxedt, '[]'); IF TG_OP='UPDATE' AND OLD.datetime_range @> NEW.datetime_range AND OLD.end_datetime_range @> NEW.end_datetime_range THEN RAISE NOTICE 'Range unchanged, not updating constraints.'; ELSE RAISE NOTICE ' SETTING CONSTRAINTS mindt: %, maxdt: % minedt: %, maxedt: % ', mindt, maxdt, minedt, maxedt; IF partition_trunc IS NULL THEN cq := format($q$ ALTER TABLE %7$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ( (datetime >= %3$L) AND (datetime <= %4$L) AND (end_datetime >= %5$L) AND (end_datetime <= %6$L) ) NOT VALID ; ALTER TABLE %7$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), mindt, maxdt, minedt, maxedt, partition_name ); ELSE cq := format($q$ ALTER TABLE %5$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %2$I CHECK ((end_datetime >= %3$L) AND (end_datetime <= %4$L)) NOT VALID ; ALTER TABLE %5$I VALIDATE CONSTRAINT %2$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), minedt, maxedt, partition_name ); END IF; RAISE NOTICE 'Altering Constraints. %', cq; EXECUTE cq; END IF; ELSE NEW.datetime_range = NULL; NEW.end_datetime_range = NULL; cq := format($q$ ALTER TABLE %3$I DROP CONSTRAINT IF EXISTS %1$I, DROP CONSTRAINT IF EXISTS %2$I, ADD CONSTRAINT %1$I CHECK ((datetime IS NULL AND end_datetime IS NULL)) NOT VALID ; ALTER TABLE %3$I VALIDATE CONSTRAINT %1$I; $q$, format('%s_dt', partition_name), format('%s_edt', partition_name), partition_name ); EXECUTE cq; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER partitions_trigger BEFORE INSERT OR UPDATE ON partitions FOR EACH ROW EXECUTE FUNCTION partitions_trigger_func(); CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac, public; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text UNIQUE NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION create_queryable_indexes() RETURNS VOID AS $$ DECLARE queryable RECORD; q text; BEGIN FOR queryable IN SELECT queryables.id as qid, CASE WHEN collections.key IS NULL THEN 'items' ELSE format('_items_%s',collections.key) END AS part, property_index_type, expression FROM queryables LEFT JOIN collections ON (collections.id = ANY (queryables.collection_ids)) JOIN LATERAL queryable(queryables.name) ON (queryables.property_index_type IS NOT NULL) LOOP q := format( $q$ CREATE INDEX IF NOT EXISTS %I ON %I USING %s ((%s)); $q$, format('%s_%s_idx', queryable.part, queryable.qid), queryable.part, COALESCE(queryable.property_index_type, 'to_text'), queryable.expression ); RAISE NOTICE '%',q; EXECUTE q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM create_queryable_indexes(); RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE TRIGGER queryables_collection_trigger AFTER INSERT OR UPDATE ON collections FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', 'https://example.org/queryables', 'type', 'object', 'title', 'Stac Queryables.', 'properties', jsonb_object_agg( name, definition ) ) FROM queryables WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR collection_ids IS NULL OR _collection_ids && collection_ids ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample int DEFAULT 5) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize bigint; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; IF _tablesample * .01 * psize < 10 THEN _tablesample := 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows', _tablesample, _collection, _partition, psize; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) WHERE q.definition IS NULL ) SELECT %L, key, jsonb_build_object('type',jsonb_typeof(value)) as definition, CASE jsonb_typeof(value) WHEN 'number' THEN 'to_float' WHEN 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample int DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; WITH ranges AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr FROM newdata n ), p AS ( SELECT collection, lower(dtr) as datetime, upper(dtr) as end_datetime, (partition_name( collection, lower(dtr) )).partition_name as name FROM ranges ) INSERT INTO partitions (collection, datetime_range, end_datetime_range) SELECT collection, tstzrange(min(datetime), max(datetime), '[]') as datetime_range, tstzrange(min(end_datetime), max(end_datetime), '[]') as end_datetime_range FROM p GROUP BY collection, name ON CONFLICT (name) DO UPDATE SET datetime_range = EXCLUDED.datetime_range, end_datetime_range = EXCLUDED.end_datetime_range ; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s JOIN deletes d USING (id, collection); DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SECURITY DEFINER SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE VIEW partition_steps AS SELECT name, date_trunc('month',lower(datetime_range)) as sdate, date_trunc('month', upper(datetime_range)) + '1 month'::interval as edate FROM partitions WHERE datetime_range IS NOT NULL AND datetime_range != 'empty'::tstzrange ORDER BY datetime_range ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default-filter-lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=quote_literal(token_rec->>_field); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); IF (SELECT count(DISTINCT _dir) FROM sorts) = 1 THEN SELECT format( '(%s) %s (%s)', concat_ws(', ', VARIADIC array_agg(quote_ident(_field))), CASE WHEN (prev AND dir = 'ASC') OR (NOT prev AND dir = 'DESC') THEN '<' ELSE '>' END, concat_ws(', ', VARIADIC array_agg(_val)) ) INTO output FROM sorts WHERE token_rec ? _field ; ELSE FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN orfilters := orfilters || format('(%s %s %s)', quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), quote_ident(sort._field), CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; andfilters := andfilters || format('%s = %s', quote_ident(sort._field), sort._val ); END LOOP; output := array_to_string(orfilters, ' OR '); END IF; DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL ; DROP FUNCTION IF EXISTS search_query; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; SELECT jsonb_agg(content) INTO out_records FROM results WHERE content is not NULL; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT set_version('0.6.9'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.0-0.7.1.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL drop function if exists "pgstac"."maintain_partition_queries"(part text, dropindexes boolean, rebuildindexes boolean); CREATE INDEX queryables_collection_idx ON pgstac.queryables USING gin (collection_ids); set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.maintain_partition_queries(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false, idxconcurrently boolean DEFAULT false) RETURNS SETOF text LANGUAGE plpgsql AS $function$ DECLARE parent text; level int; isleaf bool; collection collections%ROWTYPE; subpart text; baseidx text; queryable_name text; queryable_property_index_type text; queryable_property_wrapper text; queryable_parsed RECORD; deletedidx pg_indexes%ROWTYPE; q text; idx text; collection_partition bigint; _concurrently text := ''; BEGIN RAISE NOTICE 'Maintaining partition: %', part; IF idxconcurrently THEN _concurrently='CONCURRENTLY'; END IF; -- Get root partition SELECT parentrelid::text, pt.isleaf, pt.level INTO parent, isleaf, level FROM pg_partition_tree('items') pt WHERE relid::text = part; IF NOT FOUND THEN RAISE NOTICE 'Partition % Does Not Exist In Partition Tree', part; RETURN; END IF; -- If this is a parent partition, recurse to leaves IF NOT isleaf THEN FOR subpart IN SELECT relid::text FROM pg_partition_tree(part) WHERE relid::text != part LOOP RAISE NOTICE 'Recursing to %', subpart; RETURN QUERY SELECT * FROM maintain_partition_queries(subpart, dropindexes, rebuildindexes); END LOOP; RETURN; -- Don't continue since not an end leaf END IF; -- Get collection collection_partition := ((regexp_match(part, E'^_items_([0-9]+)'))[1])::bigint; RAISE NOTICE 'COLLECTION PARTITION: %', collection_partition; SELECT * INTO STRICT collection FROM collections WHERE key = collection_partition; RAISE NOTICE 'COLLECTION ID: %s', collection.id; -- Create temp table with existing indexes CREATE TEMP TABLE existing_indexes ON COMMIT DROP AS SELECT * FROM pg_indexes WHERE schemaname='pgstac' AND tablename=part; -- Check if index exists for each queryable. FOR queryable_name, queryable_property_index_type, queryable_property_wrapper IN SELECT name, COALESCE(property_index_type, 'BTREE'), COALESCE(property_wrapper, 'to_text') FROM queryables WHERE name NOT in ('id', 'datetime', 'geometry') AND ( collection_ids IS NULL OR collection_ids = '{}'::text[] OR collection.id = ANY (collection_ids) ) LOOP baseidx := format( $q$ ON %I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, part, queryable_property_index_type, queryable_property_wrapper, queryable_name ); RAISE NOTICE 'BASEIDX: %', baseidx; RAISE NOTICE 'IDXSEARCH: %', format($q$[(']%s[')]$q$, queryable_name); -- If index already exists, delete it from existing indexes type table FOR deletedidx IN DELETE FROM existing_indexes WHERE indexdef ~* format($q$[(']%s[')]$q$, queryable_name) RETURNING * LOOP RAISE NOTICE 'EXISTING INDEX: %', deletedidx; IF NOT FOUND THEN -- index did not exist, create it RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); ELSIF rebuildindexes THEN RETURN NEXT format('REINDEX %I %s;', deletedidx.indexname, _concurrently); END IF; END LOOP; IF NOT FOUND THEN RAISE NOTICE 'CREATING INDEX for %', queryable_name; RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); END IF; END LOOP; IF NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part, '_datetime_end_datetime_idx')) THEN RETURN NEXT format( $f$CREATE INDEX IF NOT EXISTS %I ON %I USING BTREE (datetime DESC, end_datetime ASC);$f$, concat(part, '_datetime_end_datetime_idx'), part ); END IF; IF NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part, '_geometry_idx')) THEN RETURN NEXT format( $f$CREATE INDEX IF NOT EXISTS %I ON %I USING GIST (geometry);$f$, concat(part, '_geometry_idx'), part ); END IF; IF NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part, '_pk')) THEN RETURN NEXT format( $f$CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I USING BTREE(id);$f$, concat(part, '_pk'), part ); END IF; DELETE FROM existing_indexes WHERE indexname::text IN ( concat(part, '_datetime_end_datetime_idx'), concat(part, '_geometry_idx'), concat(part, '_pk') ); -- Remove indexes that were not expected FOR idx IN SELECT indexname::text FROM existing_indexes LOOP RAISE WARNING 'Index: % is not defined by queryables.', idx; IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', idx); END IF; END LOOP; DROP TABLE existing_indexes; RAISE NOTICE 'Returning from maintain_partition_queries.'; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.queryable_signature(n text, c text[]) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT concat(n, c); $function$ ; CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables'; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM collections LEFT JOIN unnest(NEW.collection_ids) c ON (collections.id = c) WHERE c IS NULL ) THEN RAISE foreign_key_violation; RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation; RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $function$ ; CREATE OR REPLACE PROCEDURE pgstac.analyze_items() LANGUAGE plpgsql AS $procedure$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $procedure$ ; CREATE OR REPLACE FUNCTION pgstac.get_token_filter(_search jsonb DEFAULT '{}'::jsonb, token_rec jsonb DEFAULT NULL::jsonb) RETURNS text LANGUAGE plpgsql SET transform_null_equals TO 'true' AS $function$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; token_item items%ROWTYPE; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); IF token_id IS NULL OR token_id = '' THEN RAISE WARNING 'next or prev set, but no token id found'; RETURN NULL; END IF; SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; token_item := jsonb_populate_record(null::items, token_rec); RAISE NOTICE 'TOKEN ITEM ----- %', token_item; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=get_token_val_str(_field, token_item); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s < %s) OR (%s IS NULL) )$f$, sort._field, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; --orfilter := format('%s IS NULL', sort._field); ELSE orfilter := format($f$( (%s > %s) OR (%s IS NULL) )$f$, sort._field, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.queue_timeout() RETURNS interval LANGUAGE sql AS $function$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' SET cursor_tuple_fraction TO '1' AS $function$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) SELECT jsonb_agg(content) INTO out_records FROM ordered; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $function$ ; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON pgstac.queryables FOR EACH ROW EXECUTE FUNCTION pgstac.queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON pgstac.queryables FOR EACH ROW WHEN (((new.name = old.name) AND (new.collection_ids IS DISTINCT FROM old.collection_ids))) EXECUTE FUNCTION pgstac.queryables_constraint_triggerfunc(); -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false') ON CONFLICT DO NOTHING ; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.0.sql ================================================ DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DROP FUNCTION IF EXISTS analyze_items; DROP FUNCTION IF EXISTS validate_constraints; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT set_config( 'statement_timeout', t2s(coalesce( get_setting('queue_timeout'), '1h' )), false )::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; DROP VIEW IF EXISTS pgstac_index; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE parent text; level int; isleaf bool; collection collections%ROWTYPE; subpart text; baseidx text; queryable_name text; queryable_property_index_type text; queryable_property_wrapper text; queryable_parsed RECORD; deletedidx pg_indexes%ROWTYPE; q text; idx text; collection_partition bigint; _concurrently text := ''; BEGIN RAISE NOTICE 'Maintaining partition: %', part; IF get_setting_bool('use_queue') THEN _concurrently='CONCURRENTLY'; END IF; -- Get root partition SELECT parentrelid::text, pt.isleaf, pt.level INTO parent, isleaf, level FROM pg_partition_tree('items') pt WHERE relid::text = part; IF NOT FOUND THEN RAISE NOTICE 'Partition % Does Not Exist In Partition Tree', part; RETURN; END IF; -- If this is a parent partition, recurse to leaves IF NOT isleaf THEN FOR subpart IN SELECT relid::text FROM pg_partition_tree(part) WHERE relid::text != part LOOP RAISE NOTICE 'Recursing to %', subpart; RETURN QUERY SELECT * FROM maintain_partition_queries(subpart, dropindexes, rebuildindexes); END LOOP; RETURN; -- Don't continue since not an end leaf END IF; -- Get collection collection_partition := ((regexp_match(part, E'^_items_([0-9]+)'))[1])::bigint; RAISE NOTICE 'COLLECTION PARTITION: %', collection_partition; SELECT * INTO STRICT collection FROM collections WHERE key = collection_partition; RAISE NOTICE 'COLLECTION ID: %s', collection.id; -- Create temp table with existing indexes CREATE TEMP TABLE existing_indexes ON COMMIT DROP AS SELECT * FROM pg_indexes WHERE schemaname='pgstac' AND tablename=part; -- Check if index exists for each queryable. FOR queryable_name, queryable_property_index_type, queryable_property_wrapper IN SELECT name, COALESCE(property_index_type, 'BTREE'), COALESCE(property_wrapper, 'to_text') FROM queryables WHERE name NOT in ('id', 'datetime', 'geometry') AND ( collection_ids IS NULL OR collection_ids = '{}'::text[] OR collection.id = ANY (collection_ids) ) UNION ALL SELECT 'datetime desc, end_datetime', 'BTREE', '' UNION ALL SELECT 'geometry', 'GIST', '' UNION ALL SELECT 'id', 'BTREE', '' LOOP baseidx := format( $q$ ON %I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, part, queryable_property_index_type, queryable_property_wrapper, queryable_name ); RAISE NOTICE 'BASEIDX: %', baseidx; RAISE NOTICE 'IDXSEARCH: %', format($q$[(']%s[')]$q$, queryable_name); -- If index already exists, delete it from existing indexes type table FOR deletedidx IN DELETE FROM existing_indexes WHERE indexdef ~* format($q$[(']%s[')]$q$, queryable_name) RETURNING * LOOP RAISE NOTICE 'EXISTING INDEX: %', deletedidx; IF NOT FOUND THEN -- index did not exist, create it RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); ELSIF rebuildindexes THEN RETURN NEXT format('REINDEX %I %s;', deletedidx.indexname, _concurrently); END IF; END LOOP; END LOOP; -- Remove indexes that were not expected FOR idx IN SELECT indexname::text FROM existing_indexes LOOP RAISE WARNING 'Index: % is not defined by queryables.', idx; IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', idx); END IF; END LOOP; DROP TABLE existing_indexes; RAISE NOTICE 'Returning from maintain_partition_queries.'; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions AS SELECT * FROM partition_sys_meta LEFT JOIN partition_stats USING (partition); CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, partitions.collection INTO cdtrange, cedtrange, collection FROM partitions WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; $q$, t, format('%s_dt', t), t, format('%s_dt', t) ); ELSE q :=format( $q$ ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; $q$, t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t) ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE literal text; BEGIN RAISE NOTICE '% %', _field, _item; CREATE TEMP TABLE _token_item ON COMMIT DROP AS SELECT (_item).*; EXECUTE format($q$ SELECT quote_literal(%s) FROM _token_item $q$, _field) INTO literal; DROP TABLE IF EXISTS _token_item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; token_item items%ROWTYPE; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; token_item := jsonb_populate_record(null::items, token_rec); RAISE NOTICE 'TOKEN ITEM ----- %', token_item; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=get_token_val_str(_field, token_item); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP RAISE NOTICE 'SORT: %', sort; IF sort._row = 1 THEN IF sort._val IS NULL THEN orfilters := orfilters || format('(%s IS NOT NULL)', sort._field); ELSE orfilters := orfilters || format('(%s %s %s)', sort._field, CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; ELSE IF sort._val IS NULL THEN orfilters := orfilters || format('(%s AND %s IS NOT NULL)', array_to_string(andfilters, ' AND '), sort._field); ELSE orfilters := orfilters || format('(%s AND %s %s %s)', array_to_string(andfilters, ' AND '), sort._field, CASE WHEN (prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC') THEN '<' ELSE '>' END, sort._val ); END IF; END IF; IF sort._val IS NULL THEN andfilters := andfilters || format('%s IS NULL', sort._field ); ELSE andfilters := andfilters || format('%s = %s', sort._field, sort._val ); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: |%|',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; -- if ids is set, short circuit and just use direct ids query for each id -- skip any paging or caching -- hard codes ordering in the same order as the array of ids IF _search ? 'ids' THEN INSERT INTO results (content) SELECT CASE WHEN _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN content_nonhydrated(items, _search->'fields') ELSE content_hydrate(items, _search->'fields') END FROM items WHERE items.id = ANY(to_text_array(_search->'ids')) AND CASE WHEN _search ? 'collections' THEN items.collection = ANY(to_text_array(_search->'collections')) ELSE TRUE END ORDER BY items.datetime desc, items.id desc ; SELECT INTO total_count count(*) FROM results; ELSE searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; END IF; WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) SELECT jsonb_agg(content) INTO out_records FROM ordered; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS analyze_items; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP RAISE NOTICE '% % %', clock_timestamp(), timeout_ts, current_setting('statement_timeout', TRUE); SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; RAISE NOTICE '%', queue_timeout(); END LOOP; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS validate_constraints; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; INSERT INTO queryables (name, definition) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}'), ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}'), ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}') ON CONFLICT DO NOTHING; INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE') ON CONFLICT DO NOTHING; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'true') ON CONFLICT DO NOTHING ; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.1-0.7.2.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.maintain_partition_queries(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false, idxconcurrently boolean DEFAULT false) RETURNS SETOF text LANGUAGE plpgsql AS $function$ DECLARE parent text; level int; isleaf bool; collection collections%ROWTYPE; subpart text; baseidx text; queryable_name text; queryable_property_index_type text; queryable_property_wrapper text; queryable_parsed RECORD; deletedidx pg_indexes%ROWTYPE; q text; idx text; collection_partition bigint; _concurrently text := ''; idxname text; BEGIN RAISE NOTICE 'Maintaining partition: %', part; IF idxconcurrently THEN _concurrently='CONCURRENTLY'; END IF; -- Get root partition SELECT parentrelid::text, pt.isleaf, pt.level INTO parent, isleaf, level FROM pg_partition_tree('items') pt WHERE relid::text = part; IF NOT FOUND THEN RAISE NOTICE 'Partition % Does Not Exist In Partition Tree', part; RETURN; END IF; -- If this is a parent partition, recurse to leaves IF NOT isleaf THEN FOR subpart IN SELECT relid::text FROM pg_partition_tree(part) WHERE relid::text != part LOOP RAISE NOTICE 'Recursing to %', subpart; RETURN QUERY SELECT * FROM maintain_partition_queries(subpart, dropindexes, rebuildindexes); END LOOP; RETURN; -- Don't continue since not an end leaf END IF; -- Get collection collection_partition := ((regexp_match(part, E'^_items_([0-9]+)'))[1])::bigint; RAISE NOTICE 'COLLECTION PARTITION: %', collection_partition; SELECT * INTO STRICT collection FROM collections WHERE key = collection_partition; RAISE NOTICE 'COLLECTION ID: %s', collection.id; -- Create temp table with existing indexes CREATE TEMP TABLE existing_indexes ON COMMIT DROP AS SELECT * FROM pg_indexes WHERE schemaname='pgstac' AND tablename=part; -- Check if index exists for each queryable. FOR queryable_name, queryable_property_index_type, queryable_property_wrapper IN SELECT name, COALESCE(property_index_type, 'BTREE'), COALESCE(property_wrapper, 'to_text') FROM queryables WHERE name NOT in ('id', 'datetime', 'geometry') AND ( collection_ids IS NULL OR collection_ids = '{}'::text[] OR collection.id = ANY (collection_ids) ) LOOP baseidx := format( $q$ ON %I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, part, queryable_property_index_type, queryable_property_wrapper, queryable_name ); RAISE NOTICE 'BASEIDX: %', baseidx; RAISE NOTICE 'IDXSEARCH: %', format($q$[(']%s[')]$q$, queryable_name); -- If index already exists, delete it from existing indexes type table FOR deletedidx IN DELETE FROM existing_indexes WHERE indexdef ~* format($q$[(']%s[')]$q$, queryable_name) RETURNING * LOOP RAISE NOTICE 'EXISTING INDEX: %', deletedidx; IF NOT FOUND THEN -- index did not exist, create it RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); ELSIF rebuildindexes THEN RETURN NEXT format('REINDEX %I %s;', deletedidx.indexname, _concurrently); END IF; END LOOP; IF NOT FOUND THEN RAISE NOTICE 'CREATING INDEX for %', queryable_name; RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); END IF; END LOOP; IF NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part, '_datetime_end_datetime_idx')) THEN RETURN NEXT format( $f$CREATE INDEX IF NOT EXISTS %I ON %I USING BTREE (datetime DESC, end_datetime ASC);$f$, concat(part, '_datetime_end_datetime_idx'), part ); END IF; IF NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part, '_geometry_idx')) THEN RETURN NEXT format( $f$CREATE INDEX IF NOT EXISTS %I ON %I USING GIST (geometry);$f$, concat(part, '_geometry_idx'), part ); END IF; FOR idxname IN SELECT indexname::text FROM pg_indexes WHERE tablename::text = part AND indexdef ILIKE 'CREATE UNIQUE INDEX % USING btree(id)' LOOP IF idxname != concat(part, '_pk') AND NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part,'_pk')) THEN RETURN NEXT format( $f$ALTER INDEX IF EXISTS %I RENAME TO %I;$f$, idxname, concat(part, '_pk') ); ELSIF EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part,'_pk')) AND dropindexes THEN RETURN NEXT format( $f$DROP INDEX IF EXISTS %I;$f$, idxname ); END IF; END LOOP; IF NOT EXISTS ( SELECT indexname::text FROM pg_indexes WHERE tablename::text = part AND indexdef ILIKE 'CREATE UNIQUE INDEX % USING btree (id)' ) THEN RETURN NEXT format( $f$CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I USING BTREE(id);$f$, concat(part, '_pk'), part ); END IF; DELETE FROM existing_indexes WHERE indexname::text IN ( concat(part, '_datetime_end_datetime_idx'), concat(part, '_geometry_idx'), concat(part, '_pk') ); -- Remove indexes that were not expected FOR idx IN SELECT indexname::text FROM existing_indexes LOOP RAISE WARNING 'Index: % is not defined by queryables.', idx; IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', idx); END IF; END LOOP; DROP TABLE existing_indexes; RAISE NOTICE 'Returning from maintain_partition_queries.'; RETURN; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false') ON CONFLICT DO NOTHING ; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.1.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables'; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM collections LEFT JOIN unnest(NEW.collection_ids) c ON (collections.id = c) WHERE c IS NULL ) THEN RAISE foreign_key_violation; RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation; RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; DROP VIEW IF EXISTS pgstac_index; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE parent text; level int; isleaf bool; collection collections%ROWTYPE; subpart text; baseidx text; queryable_name text; queryable_property_index_type text; queryable_property_wrapper text; queryable_parsed RECORD; deletedidx pg_indexes%ROWTYPE; q text; idx text; collection_partition bigint; _concurrently text := ''; BEGIN RAISE NOTICE 'Maintaining partition: %', part; IF idxconcurrently THEN _concurrently='CONCURRENTLY'; END IF; -- Get root partition SELECT parentrelid::text, pt.isleaf, pt.level INTO parent, isleaf, level FROM pg_partition_tree('items') pt WHERE relid::text = part; IF NOT FOUND THEN RAISE NOTICE 'Partition % Does Not Exist In Partition Tree', part; RETURN; END IF; -- If this is a parent partition, recurse to leaves IF NOT isleaf THEN FOR subpart IN SELECT relid::text FROM pg_partition_tree(part) WHERE relid::text != part LOOP RAISE NOTICE 'Recursing to %', subpart; RETURN QUERY SELECT * FROM maintain_partition_queries(subpart, dropindexes, rebuildindexes); END LOOP; RETURN; -- Don't continue since not an end leaf END IF; -- Get collection collection_partition := ((regexp_match(part, E'^_items_([0-9]+)'))[1])::bigint; RAISE NOTICE 'COLLECTION PARTITION: %', collection_partition; SELECT * INTO STRICT collection FROM collections WHERE key = collection_partition; RAISE NOTICE 'COLLECTION ID: %s', collection.id; -- Create temp table with existing indexes CREATE TEMP TABLE existing_indexes ON COMMIT DROP AS SELECT * FROM pg_indexes WHERE schemaname='pgstac' AND tablename=part; -- Check if index exists for each queryable. FOR queryable_name, queryable_property_index_type, queryable_property_wrapper IN SELECT name, COALESCE(property_index_type, 'BTREE'), COALESCE(property_wrapper, 'to_text') FROM queryables WHERE name NOT in ('id', 'datetime', 'geometry') AND ( collection_ids IS NULL OR collection_ids = '{}'::text[] OR collection.id = ANY (collection_ids) ) LOOP baseidx := format( $q$ ON %I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, part, queryable_property_index_type, queryable_property_wrapper, queryable_name ); RAISE NOTICE 'BASEIDX: %', baseidx; RAISE NOTICE 'IDXSEARCH: %', format($q$[(']%s[')]$q$, queryable_name); -- If index already exists, delete it from existing indexes type table FOR deletedidx IN DELETE FROM existing_indexes WHERE indexdef ~* format($q$[(']%s[')]$q$, queryable_name) RETURNING * LOOP RAISE NOTICE 'EXISTING INDEX: %', deletedidx; IF NOT FOUND THEN -- index did not exist, create it RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); ELSIF rebuildindexes THEN RETURN NEXT format('REINDEX %I %s;', deletedidx.indexname, _concurrently); END IF; END LOOP; IF NOT FOUND THEN RAISE NOTICE 'CREATING INDEX for %', queryable_name; RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); END IF; END LOOP; IF NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part, '_datetime_end_datetime_idx')) THEN RETURN NEXT format( $f$CREATE INDEX IF NOT EXISTS %I ON %I USING BTREE (datetime DESC, end_datetime ASC);$f$, concat(part, '_datetime_end_datetime_idx'), part ); END IF; IF NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part, '_geometry_idx')) THEN RETURN NEXT format( $f$CREATE INDEX IF NOT EXISTS %I ON %I USING GIST (geometry);$f$, concat(part, '_geometry_idx'), part ); END IF; IF NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part, '_pk')) THEN RETURN NEXT format( $f$CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I USING BTREE(id);$f$, concat(part, '_pk'), part ); END IF; DELETE FROM existing_indexes WHERE indexname::text IN ( concat(part, '_datetime_end_datetime_idx'), concat(part, '_geometry_idx'), concat(part, '_pk') ); -- Remove indexes that were not expected FOR idx IN SELECT indexname::text FROM existing_indexes LOOP RAISE WARNING 'Index: % is not defined by queryables.', idx; IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', idx); END IF; END LOOP; DROP TABLE existing_indexes; RAISE NOTICE 'Returning from maintain_partition_queries.'; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions AS SELECT * FROM partition_sys_meta LEFT JOIN partition_stats USING (partition); CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, partitions.collection INTO cdtrange, cedtrange, collection FROM partitions WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; $q$, t, format('%s_dt', t), t, format('%s_dt', t) ); ELSE q :=format( $q$ ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; $q$, t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t) ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE literal text; BEGIN RAISE NOTICE '% %', _field, _item; CREATE TEMP TABLE _token_item ON COMMIT DROP AS SELECT (_item).*; EXECUTE format($q$ SELECT quote_literal(%s) FROM _token_item $q$, _field) INTO literal; DROP TABLE IF EXISTS _token_item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; token_item items%ROWTYPE; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); IF token_id IS NULL OR token_id = '' THEN RAISE WARNING 'next or prev set, but no token id found'; RETURN NULL; END IF; SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; token_item := jsonb_populate_record(null::items, token_rec); RAISE NOTICE 'TOKEN ITEM ----- %', token_item; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=get_token_val_str(_field, token_item); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s < %s) OR (%s IS NULL) )$f$, sort._field, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; --orfilter := format('%s IS NULL', sort._field); ELSE orfilter := format($f$( (%s > %s) OR (%s IS NULL) )$f$, sort._field, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) SELECT jsonb_agg(content) INTO out_records FROM ordered; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false') ON CONFLICT DO NOTHING ; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.10-0.8.0.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL alter table "pgstac"."collections" add column "datetime" timestamp with time zone generated always as (collection_datetime(content)) stored; alter table "pgstac"."collections" add column "end_datetime" timestamp with time zone generated always as (collection_enddatetime(content)) stored; alter table "pgstac"."collections" add column "geometry" geometry generated always as (collection_geom(content)) stored; alter table "pgstac"."collections" add column "private" jsonb; alter table "pgstac"."items" add column "private" jsonb; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.collection_datetime(content jsonb) RETURNS timestamp with time zone LANGUAGE sql IMMUTABLE STRICT AS $function$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_enddatetime(content jsonb) RETURNS timestamp with time zone LANGUAGE sql IMMUTABLE STRICT AS $function$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_geom(content jsonb) RETURNS geometry LANGUAGE sql IMMUTABLE STRICT AS $function$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $function$ ; CREATE OR REPLACE FUNCTION pgstac.content_dehydrate(content jsonb) RETURNS items LANGUAGE sql STABLE AS $function$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.10.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; BEGIN SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); init_limit int := _limit; _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; items_cnt int := coalesce(jsonb_array_length(_search->'ids'),0); BEGIN RAISE NOTICE 'Items Count: %', items_cnt; IF items_cnt > 0 THEN IF items_cnt <= _limit THEN _limit := items_cnt - 1; ELSE _limit := items_cnt; END IF; RAISE NOTICE 'Items is set. Changing limit to %', items_cnt; END IF; searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = init_limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', init_limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', init_limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.10'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.2-0.7.3.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL drop view if exists "pgstac"."pgstac_indexes"; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.indexdef(q pgstac.queryables) RETURNS text LANGUAGE plpgsql IMMUTABLE AS $function$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.normalize_indexdef(def text) RETURNS text LANGUAGE plpgsql IMMUTABLE PARALLEL SAFE STRICT AS $function$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.unnest_collection(collection_ids text[] DEFAULT NULL::text[]) RETURNS SETOF text LANGUAGE plpgsql STABLE AS $function$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.maintain_partition_queries(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false, idxconcurrently boolean DEFAULT false) RETURNS SETOF text LANGUAGE plpgsql AS $function$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $function$ ; create or replace view "pgstac"."pgstac_indexes" as SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(i.indexdef, (i.indexname)::text, ''::text), 'pgstac.'::text, ''::text), ' \t\n'::text), '[ ]+'::text, ' '::text, 'g'::text) AS idx, COALESCE((regexp_match(i.indexdef, '\(([a-zA-Z]+)\)'::text))[1], (regexp_match(i.indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'::text))[1], CASE WHEN (i.indexdef ~* '\(datetime desc, end_datetime\)'::text) THEN 'datetime'::text ELSE NULL::text END) AS field, pg_table_size(((i.indexname)::text)::regclass) AS index_size, pg_size_pretty(pg_table_size(((i.indexname)::text)::regclass)) AS index_size_pretty FROM pg_indexes i WHERE ((i.schemaname = 'pgstac'::name) AND (i.tablename ~ '_items_'::text) AND (i.indexdef !~* ' only '::text)); -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false') ON CONFLICT DO NOTHING ; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.2.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables'; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM collections LEFT JOIN unnest(NEW.collection_ids) c ON (collections.id = c) WHERE c IS NULL ) THEN RAISE foreign_key_violation; RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation; RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; DROP VIEW IF EXISTS pgstac_index; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE parent text; level int; isleaf bool; collection collections%ROWTYPE; subpart text; baseidx text; queryable_name text; queryable_property_index_type text; queryable_property_wrapper text; queryable_parsed RECORD; deletedidx pg_indexes%ROWTYPE; q text; idx text; collection_partition bigint; _concurrently text := ''; idxname text; BEGIN RAISE NOTICE 'Maintaining partition: %', part; IF idxconcurrently THEN _concurrently='CONCURRENTLY'; END IF; -- Get root partition SELECT parentrelid::text, pt.isleaf, pt.level INTO parent, isleaf, level FROM pg_partition_tree('items') pt WHERE relid::text = part; IF NOT FOUND THEN RAISE NOTICE 'Partition % Does Not Exist In Partition Tree', part; RETURN; END IF; -- If this is a parent partition, recurse to leaves IF NOT isleaf THEN FOR subpart IN SELECT relid::text FROM pg_partition_tree(part) WHERE relid::text != part LOOP RAISE NOTICE 'Recursing to %', subpart; RETURN QUERY SELECT * FROM maintain_partition_queries(subpart, dropindexes, rebuildindexes); END LOOP; RETURN; -- Don't continue since not an end leaf END IF; -- Get collection collection_partition := ((regexp_match(part, E'^_items_([0-9]+)'))[1])::bigint; RAISE NOTICE 'COLLECTION PARTITION: %', collection_partition; SELECT * INTO STRICT collection FROM collections WHERE key = collection_partition; RAISE NOTICE 'COLLECTION ID: %s', collection.id; -- Create temp table with existing indexes CREATE TEMP TABLE existing_indexes ON COMMIT DROP AS SELECT * FROM pg_indexes WHERE schemaname='pgstac' AND tablename=part; -- Check if index exists for each queryable. FOR queryable_name, queryable_property_index_type, queryable_property_wrapper IN SELECT name, COALESCE(property_index_type, 'BTREE'), COALESCE(property_wrapper, 'to_text') FROM queryables WHERE name NOT in ('id', 'datetime', 'geometry') AND ( collection_ids IS NULL OR collection_ids = '{}'::text[] OR collection.id = ANY (collection_ids) ) LOOP baseidx := format( $q$ ON %I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, part, queryable_property_index_type, queryable_property_wrapper, queryable_name ); RAISE NOTICE 'BASEIDX: %', baseidx; RAISE NOTICE 'IDXSEARCH: %', format($q$[(']%s[')]$q$, queryable_name); -- If index already exists, delete it from existing indexes type table FOR deletedidx IN DELETE FROM existing_indexes WHERE indexdef ~* format($q$[(']%s[')]$q$, queryable_name) RETURNING * LOOP RAISE NOTICE 'EXISTING INDEX: %', deletedidx; IF NOT FOUND THEN -- index did not exist, create it RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); ELSIF rebuildindexes THEN RETURN NEXT format('REINDEX %I %s;', deletedidx.indexname, _concurrently); END IF; END LOOP; IF NOT FOUND THEN RAISE NOTICE 'CREATING INDEX for %', queryable_name; RETURN NEXT format('CREATE INDEX %s %s;', _concurrently, baseidx); END IF; END LOOP; IF NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part, '_datetime_end_datetime_idx')) THEN RETURN NEXT format( $f$CREATE INDEX IF NOT EXISTS %I ON %I USING BTREE (datetime DESC, end_datetime ASC);$f$, concat(part, '_datetime_end_datetime_idx'), part ); END IF; IF NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part, '_geometry_idx')) THEN RETURN NEXT format( $f$CREATE INDEX IF NOT EXISTS %I ON %I USING GIST (geometry);$f$, concat(part, '_geometry_idx'), part ); END IF; FOR idxname IN SELECT indexname::text FROM pg_indexes WHERE tablename::text = part AND indexdef ILIKE 'CREATE UNIQUE INDEX % USING btree(id)' LOOP IF idxname != concat(part, '_pk') AND NOT EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part,'_pk')) THEN RETURN NEXT format( $f$ALTER INDEX IF EXISTS %I RENAME TO %I;$f$, idxname, concat(part, '_pk') ); ELSIF EXISTS (SELECT * FROM pg_indexes WHERE indexname::text = concat(part,'_pk')) AND dropindexes THEN RETURN NEXT format( $f$DROP INDEX IF EXISTS %I;$f$, idxname ); END IF; END LOOP; IF NOT EXISTS ( SELECT indexname::text FROM pg_indexes WHERE tablename::text = part AND indexdef ILIKE 'CREATE UNIQUE INDEX % USING btree (id)' ) THEN RETURN NEXT format( $f$CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I USING BTREE(id);$f$, concat(part, '_pk'), part ); END IF; DELETE FROM existing_indexes WHERE indexname::text IN ( concat(part, '_datetime_end_datetime_idx'), concat(part, '_geometry_idx'), concat(part, '_pk') ); -- Remove indexes that were not expected FOR idx IN SELECT indexname::text FROM existing_indexes LOOP RAISE WARNING 'Index: % is not defined by queryables.', idx; IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', idx); END IF; END LOOP; DROP TABLE existing_indexes; RAISE NOTICE 'Returning from maintain_partition_queries.'; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions AS SELECT * FROM partition_sys_meta LEFT JOIN partition_stats USING (partition); CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, partitions.collection INTO cdtrange, cedtrange, collection FROM partitions WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; $q$, t, format('%s_dt', t), t, format('%s_dt', t) ); ELSE q :=format( $q$ ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; $q$, t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t) ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE literal text; BEGIN RAISE NOTICE '% %', _field, _item; CREATE TEMP TABLE _token_item ON COMMIT DROP AS SELECT (_item).*; EXECUTE format($q$ SELECT quote_literal(%s) FROM _token_item $q$, _field) INTO literal; DROP TABLE IF EXISTS _token_item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; token_item items%ROWTYPE; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); IF token_id IS NULL OR token_id = '' THEN RAISE WARNING 'next or prev set, but no token id found'; RETURN NULL; END IF; SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; token_item := jsonb_populate_record(null::items, token_rec); RAISE NOTICE 'TOKEN ITEM ----- %', token_item; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=get_token_val_str(_field, token_item); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s < %s) OR (%s IS NULL) )$f$, sort._field, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; --orfilter := format('%s IS NULL', sort._field); ELSE orfilter := format($f$( (%s > %s) OR (%s IS NULL) )$f$, sort._field, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) SELECT jsonb_agg(content) INTO out_records FROM ordered; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false') ON CONFLICT DO NOTHING ; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.3-0.7.4.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL drop function if exists "pgstac"."get_token_filter"(_search jsonb, token_rec jsonb); create unlogged table "pgstac"."format_item_cache" ( "id" text not null, "collection" text not null, "fields" text not null, "hydrated" boolean not null, "output" jsonb, "lastused" timestamp with time zone default now(), "usecount" integer default 1, "timetoformat" double precision ); CREATE INDEX format_item_cache_lastused_idx ON pgstac.format_item_cache USING btree (lastused); CREATE UNIQUE INDEX format_item_cache_pkey ON pgstac.format_item_cache USING btree (collection, id, fields, hydrated); alter table "pgstac"."format_item_cache" add constraint "format_item_cache_pkey" PRIMARY KEY using index "format_item_cache_pkey"; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.age_ms(a timestamp with time zone, b timestamp with time zone DEFAULT clock_timestamp()) RETURNS double precision LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT abs(extract(epoch from age(a,b)) * 1000); $function$ ; CREATE OR REPLACE FUNCTION pgstac.format_item(_item pgstac.items, _fields jsonb DEFAULT '{}'::jsonb, _hydrated boolean DEFAULT true) RETURNS jsonb LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_token_filter(_sortby jsonb DEFAULT '[{"field": "datetime", "direction": "desc"}]'::jsonb, token_item pgstac.items DEFAULT NULL::pgstac.items, prev boolean DEFAULT false, inclusive boolean DEFAULT false) RETURNS text LANGUAGE plpgsql SET transform_null_equals TO 'true' AS $function$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_token_record(_token text, OUT prev boolean, OUT item pgstac.items) RETURNS record LANGUAGE plpgsql STABLE STRICT AS $function$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_rows(_where text DEFAULT 'TRUE'::text, _orderby text DEFAULT 'datetime DESC, id DESC'::text, partitions text[] DEFAULT NULL::text[], _limit integer DEFAULT 10) RETURNS SETOF pgstac.items LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' AS $function$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %', sdate, edate; query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go.', n, _limit, sdate, edate, records_left; IF records_left <= 0 THEN RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %', sdate, edate; query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go.', n, _limit, sdate, edate, records_left; IF records_left <= 0 THEN RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; RETURN QUERY EXECUTE query; END IF; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_token_val_str(_field text, _item pgstac.items) RETURNS text LANGUAGE plpgsql AS $function$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.partition_after_triggerfunc() RETURNS trigger LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM collections LEFT JOIN unnest(NEW.collection_ids) c ON (collections.id = c) WHERE c IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: %', token_where; IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.sort_sqlorderby(_search jsonb DEFAULT NULL::jsonb, reverse boolean DEFAULT false) RETURNS text LANGUAGE sql AS $function$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.3.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables'; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM collections LEFT JOIN unnest(NEW.collection_ids) c ON (collections.id = c) WHERE c IS NULL ) THEN RAISE foreign_key_violation; RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation; RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions AS SELECT * FROM partition_sys_meta LEFT JOIN partition_stats USING (partition); CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, partitions.collection INTO cdtrange, cedtrange, collection FROM partitions WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( -- field_orderby((items_path(value->>'field')).path_txt), (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE literal text; BEGIN RAISE NOTICE '% %', _field, _item; CREATE TEMP TABLE _token_item ON COMMIT DROP AS SELECT (_item).*; EXECUTE format($q$ SELECT quote_literal(%s) FROM _token_item $q$, _field) INTO literal; DROP TABLE IF EXISTS _token_item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_filter(_search jsonb = '{}'::jsonb, token_rec jsonb DEFAULT NULL) RETURNS text AS $$ DECLARE token_id text; filters text[] := '{}'::text[]; prev boolean := TRUE; field text; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; token_item items%ROWTYPE; BEGIN RAISE NOTICE 'Getting Token Filter. % %', _search, token_rec; -- If no token provided return NULL IF token_rec IS NULL THEN IF NOT (_search ? 'token' AND ( (_search->>'token' ILIKE 'prev:%') OR (_search->>'token' ILIKE 'next:%') ) ) THEN RETURN NULL; END IF; prev := (_search->>'token' ILIKE 'prev:%'); token_id := substr(_search->>'token', 6); IF token_id IS NULL OR token_id = '' THEN RAISE WARNING 'next or prev set, but no token id found'; RETURN NULL; END IF; SELECT to_jsonb(items) INTO token_rec FROM items WHERE id=token_id; END IF; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; RAISE NOTICE 'TOKEN ID: % %', token_rec, token_rec->'id'; token_item := jsonb_populate_record(null::items, token_rec); RAISE NOTICE 'TOKEN ITEM ----- %', token_item; CREATE TEMP TABLE sorts ( _row int GENERATED ALWAYS AS IDENTITY NOT NULL, _field text PRIMARY KEY, _dir text NOT NULL, _val text ) ON COMMIT DROP; -- Make sure we only have distinct columns to sort with taking the first one we get INSERT INTO sorts (_field, _dir) SELECT (queryable(value->>'field')).expression, get_sort_dir(value) FROM jsonb_array_elements(coalesce(_search->'sortby','[{"field":"datetime","direction":"desc"}]')) ON CONFLICT DO NOTHING ; RAISE NOTICE 'sorts 1: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); -- Get the first sort direction provided. As the id is a primary key, if there are any -- sorts after id they won't do anything, so make sure that id is the last sort item. SELECT _dir INTO dir FROM sorts ORDER BY _row ASC LIMIT 1; IF EXISTS (SELECT 1 FROM sorts WHERE _field = 'id') THEN DELETE FROM sorts WHERE _row > (SELECT _row FROM sorts WHERE _field = 'id' ORDER BY _row ASC); ELSE INSERT INTO sorts (_field, _dir) VALUES ('id', dir); END IF; -- Add value from looked up item to the sorts table UPDATE sorts SET _val=get_token_val_str(_field, token_item); -- Check if all sorts are the same direction and use row comparison -- to filter RAISE NOTICE 'sorts 2: %', (SELECT jsonb_agg(to_json(sorts)) FROM sorts); FOR sort IN SELECT * FROM sorts ORDER BY _row asc LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s < %s) OR (%s IS NULL) )$f$, sort._field, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; --orfilter := format('%s IS NULL', sort._field); ELSE orfilter := format($f$( (%s > %s) OR (%s IS NULL) )$f$, sort._field, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); DROP TABLE IF EXISTS sorts; token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; token_where text; full_where text; orderby text; query text; token_type text := substr(_search->>'token',1,4); _limit int := coalesce((_search->>'limit')::int, 10); curs refcursor; cntr int := 0; iter_record items%ROWTYPE; first_record jsonb; first_item items%ROWTYPE; last_item items%ROWTYPE; last_record jsonb; out_records jsonb := '[]'::jsonb; prev_query text; next text; prev_id text; has_next boolean := false; has_prev boolean := false; prev text; total_count bigint; context jsonb; collection jsonb; includes text[]; excludes text[]; exit_flag boolean := FALSE; batches int := 0; timer timestamptz := clock_timestamp(); pstart timestamptz; pend timestamptz; pcurs refcursor; search_where search_wheres%ROWTYPE; id text; BEGIN CREATE TEMP TABLE results (i int GENERATED ALWAYS AS IDENTITY, content jsonb) ON COMMIT DROP; searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); IF token_type='prev' THEN token_where := get_token_filter(_search, null::jsonb); orderby := sort_sqlorderby(_search, TRUE); END IF; IF token_type='next' THEN token_where := get_token_filter(_search, null::jsonb); END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL QUERY % %', full_where, clock_timestamp()-timer; timer := clock_timestamp(); FOR query IN SELECT partition_queries(full_where, orderby, search_where.partitions) LOOP timer := clock_timestamp(); query := format('%s LIMIT %s', query, _limit + 1); RAISE NOTICE 'Partition Query: %', query; batches := batches + 1; -- curs = create_cursor(query); RAISE NOTICE 'cursor_tuple_fraction: %', current_setting('cursor_tuple_fraction'); OPEN curs FOR EXECUTE query; LOOP FETCH curs into iter_record; EXIT WHEN NOT FOUND; cntr := cntr + 1; IF _search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true THEN last_record := content_nonhydrated(iter_record, _search->'fields'); ELSE last_record := content_hydrate(iter_record, _search->'fields'); END IF; last_item := iter_record; IF cntr = 1 THEN first_item := last_item; first_record := last_record; END IF; IF cntr <= _limit THEN INSERT INTO results (content) VALUES (last_record); ELSIF cntr > _limit THEN has_next := true; exit_flag := true; EXIT; END IF; END LOOP; CLOSE curs; RAISE NOTICE 'Query took %.', clock_timestamp()-timer; timer := clock_timestamp(); EXIT WHEN exit_flag; END LOOP; RAISE NOTICE 'Scanned through % partitions.', batches; WITH ordered AS (SELECT * FROM results WHERE content IS NOT NULL ORDER BY i) SELECT jsonb_agg(content) INTO out_records FROM ordered; DROP TABLE results; -- Flip things around if this was the result of a prev token query IF token_type='prev' THEN out_records := flip_jsonb_array(out_records); first_item := last_item; first_record := last_record; END IF; -- If this query has a token, see if there is data before the first record IF _search ? 'token' THEN prev_query := format( 'SELECT 1 FROM items WHERE %s LIMIT 1', concat_ws( ' AND ', _where, trim(get_token_filter(_search, to_jsonb(first_item))) ) ); RAISE NOTICE 'Query to get previous record: % --- %', prev_query, first_record; EXECUTE prev_query INTO has_prev; IF FOUND and has_prev IS NOT NULL THEN RAISE NOTICE 'Query results from prev query: %', has_prev; has_prev := TRUE; END IF; END IF; has_prev := COALESCE(has_prev, FALSE); IF has_prev THEN prev := out_records->0->>'id'; END IF; IF has_next OR token_type='prev' THEN next := out_records->-1->>'id'; END IF; IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RETURN collection; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET cursor_tuple_fraction TO 1; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false') ON CONFLICT DO NOTHING ; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.4-0.7.5.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL drop view if exists "pgstac"."partition_steps"; drop view if exists "pgstac"."partitions"; set check_function_bodies = off; create or replace view "pgstac"."partitions_view" as SELECT (pg_partition_tree.relid)::text AS partition, replace(replace( CASE WHEN (pg_partition_tree.level = 1) THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, pg_partition_tree.level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_edtrange, partition_stats.dtrange, partition_stats.edtrange, partition_stats.spatial, partition_stats.last_updated FROM ((((pg_partition_tree('pgstac.items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) LEFT JOIN pg_constraint edt ON (((edt.conrelid = c.oid) AND (edt.contype = 'c'::"char")))) LEFT JOIN pgstac.partition_stats ON (((pg_partition_tree.relid)::text = partition_stats.partition))) WHERE pg_partition_tree.isleaf; CREATE OR REPLACE FUNCTION pgstac.check_partition(_collection text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_extent(_collection text, runupdate boolean DEFAULT false) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.drop_table_constraints(t text) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_token_filter(_sortby jsonb DEFAULT '[{"field": "datetime", "direction": "desc"}]'::jsonb, token_item pgstac.items DEFAULT NULL::pgstac.items, prev boolean DEFAULT false, inclusive boolean DEFAULT false) RETURNS text LANGUAGE plpgsql SET transform_null_equals TO 'true' AS $function$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $function$ ; create materialized view "pgstac"."partition_steps" as SELECT partitions_view.partition AS name, date_trunc('month'::text, lower(partitions_view.partition_dtrange)) AS sdate, (date_trunc('month'::text, upper(partitions_view.partition_dtrange)) + '1 mon'::interval) AS edate FROM pgstac.partitions_view WHERE ((partitions_view.partition_dtrange IS NOT NULL) AND (partitions_view.partition_dtrange <> 'empty'::tstzrange)) ORDER BY partitions_view.dtrange; create materialized view "pgstac"."partitions" as SELECT partitions_view.partition, partitions_view.collection, partitions_view.level, partitions_view.reltuples, partitions_view.relhastriggers, partitions_view.partition_dtrange, partitions_view.constraint_dtrange, partitions_view.constraint_edtrange, partitions_view.dtrange, partitions_view.edtrange, partitions_view.spatial, partitions_view.last_updated FROM pgstac.partitions_view; CREATE OR REPLACE FUNCTION pgstac.repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT false) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: %', token_where; IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_rows(_where text DEFAULT 'TRUE'::text, _orderby text DEFAULT 'datetime DESC, id DESC'::text, partitions text[] DEFAULT NULL::text[], _limit integer DEFAULT 10) RETURNS SETOF pgstac.items LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' AS $function$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_partition_stats(_partition text, istrigger boolean DEFAULT false) RETURNS void LANGUAGE plpgsql STRICT AS $function$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $function$ ; CREATE UNIQUE INDEX partitions_partition_idx ON pgstac.partitions USING btree (partition); -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.4.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM collections LEFT JOIN unnest(NEW.collection_ids) c ON (collections.id = c) WHERE c IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions AS SELECT * FROM partition_sys_meta LEFT JOIN partition_stats USING (partition); CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, partitions.collection INTO cdtrange, cedtrange, collection FROM partitions WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %', sdate, edate; query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go.', n, _limit, sdate, edate, records_left; IF records_left <= 0 THEN RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %', sdate, edate; query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go.', n, _limit, sdate, edate, records_left; IF records_left <= 0 THEN RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; RETURN QUERY EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: %', token_where; IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions; SELECT set_version('0.7.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.5-0.7.6.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.6'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.5.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM collections LEFT JOIN unnest(NEW.collection_ids) c ON (collections.id = c) WHERE c IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: %', token_where; IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.6-0.7.7.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); init_limit int := _limit; _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; items_cnt int := coalesce(jsonb_array_length(_search->'ids'),0); BEGIN RAISE NOTICE 'Items Count: %', items_cnt; IF items_cnt > 0 THEN IF items_cnt <= _limit THEN _limit := items_cnt - 1; ELSE _limit := items_cnt; END IF; RAISE NOTICE 'Items is set. Changing limit to %', items_cnt; END IF; searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = init_limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', init_limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', init_limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) RETURNS pgstac.searches LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; BEGIN SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; RETURN search; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.7'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.6.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; BEGIN SELECT * INTO search FROM searches WHERE hash=search_hash(_search, _metadata) FOR UPDATE; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); search.lastused := now(); search.usecount := coalesce(search.usecount, 0) + 1; INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, search.lastused, search.usecount, _metadata) ON CONFLICT (hash) DO UPDATE SET _where = EXCLUDED._where, orderby = EXCLUDED.orderby, lastused = EXCLUDED.lastused, usecount = EXCLUDED.usecount, metadata = EXCLUDED.metadata RETURNING * INTO search ; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: %', token_where; IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.6'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.7-0.7.8.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) RETURNS pgstac.searches LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; BEGIN SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; RETURN search; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.8'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.7.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; BEGIN SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); init_limit int := _limit; _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; items_cnt int := coalesce(jsonb_array_length(_search->'ids'),0); BEGIN RAISE NOTICE 'Items Count: %', items_cnt; IF items_cnt > 0 THEN IF items_cnt <= _limit THEN _limit := items_cnt - 1; ELSE _limit := items_cnt; END IF; RAISE NOTICE 'Items is set. Changing limit to %', items_cnt; END IF; searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = init_limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', init_limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', init_limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.7'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.8-0.7.9.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.9'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.8.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; BEGIN SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); init_limit int := _limit; _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; items_cnt int := coalesce(jsonb_array_length(_search->'ids'),0); BEGIN RAISE NOTICE 'Items Count: %', items_cnt; IF items_cnt > 0 THEN IF items_cnt <= _limit THEN _limit := items_cnt - 1; ELSE _limit := items_cnt; END IF; RAISE NOTICE 'Items is set. Changing limit to %', items_cnt; END IF; searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = init_limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', init_limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', init_limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.8'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.9-0.7.10.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.collection_delete_trigger_func() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.all_collections() RETURNS jsonb LANGUAGE sql SET search_path TO 'pgstac', 'public' AS $function$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_setting_bool(_setting text, conf jsonb DEFAULT NULL::jsonb) RETURNS boolean LANGUAGE sql AS $function$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $function$ ; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON pgstac.collections FOR EACH ROW EXECUTE FUNCTION pgstac.collection_delete_trigger_func(); -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.10'); ================================================ FILE: src/pgstac/migrations/pgstac.0.7.9.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( conf->>_setting, current_setting(concat('pgstac.',_setting), TRUE), (SELECT value FROM pgstac.pgstac_settings WHERE name=_setting), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT jsonb_agg(content) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; BEGIN SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); init_limit int := _limit; _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; items_cnt int := coalesce(jsonb_array_length(_search->'ids'),0); BEGIN RAISE NOTICE 'Items Count: %', items_cnt; IF items_cnt > 0 THEN IF items_cnt <= _limit THEN _limit := items_cnt - 1; ELSE _limit := items_cnt; END IF; RAISE NOTICE 'Items is set. Changing limit to %', items_cnt; END IF; searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = init_limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', init_limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', init_limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('eo:cloud_cover','{"$ref": "https://stac-extensions.github.io/eo/v1.0.0/schema.json#/definitions/fieldsproperties/eo:cloud_cover"}','to_int','BTREE'); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.7.9'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.0-0.8.1.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.0.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; BEGIN SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.1-0.8.2.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL alter table "pgstac"."collections" alter column "id" set not null; set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.additional_properties() RETURNS boolean LANGUAGE sql AS $function$ SELECT pgstac.get_setting_bool('additional_properties'); $function$ ; CREATE OR REPLACE FUNCTION pgstac.base_url(conf jsonb DEFAULT NULL::jsonb) RETURNS text LANGUAGE sql AS $function$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql STABLE PARALLEL SAFE AS $function$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); IF _limit <= number_matched THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset = 0 THEN -- no previous paging links := jsonb_build_array( jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); ELSE links := jsonb_build_array( jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ), jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'context', jsonb_build_object( 'limit', _limit, 'matched', number_matched, 'returned', number_returned ), 'links', links ); RETURN ret; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_search_matched(_search jsonb DEFAULT '{}'::jsonb, OUT matched bigint) RETURNS bigint LANGUAGE plpgsql STABLE PARALLEL SAFE AS $function$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_search_rows(_search jsonb DEFAULT '{}'::jsonb) RETURNS SETOF jsonb LANGUAGE plpgsql AS $function$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $function$ ; create or replace view "pgstac"."collections_asitems" as SELECT collections.id, collections.geometry, 'collections'::text AS collection, collections.datetime, collections.end_datetime, jsonb_build_object('properties', (collections.content - '{links,assets,stac_version,stac_extensions}'::text), 'links', (collections.content -> 'links'::text), 'assets', (collections.content -> 'assets'::text), 'stac_version', (collections.content -> 'stac_version'::text), 'stac_extensions', (collections.content -> 'stac_extensions'::text)) AS content, collections.content AS collectionjson FROM collections; CREATE OR REPLACE FUNCTION pgstac.maintain_index(indexname text, queryable_idx text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false, idxconcurrently boolean DEFAULT false) RETURNS void LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.queryable_indexes(treeroot text DEFAULT 'items'::text, changes boolean DEFAULT false, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text) RETURNS SETOF record LANGUAGE sql AS $function$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.readonly(conf jsonb DEFAULT NULL::jsonb) RETURNS boolean LANGUAGE sql AS $function$ SELECT pgstac.get_setting_bool('readonly', conf); $function$ ; CREATE OR REPLACE FUNCTION pgstac.check_partition(_collection text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, wrapper text DEFAULT NULL::text) RETURNS text LANGUAGE plpgsql STABLE AS $function$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.geometrysearch(geom geometry, queryhash text, fields jsonb DEFAULT NULL::jsonb, _scanlimit integer DEFAULT 10000, _limit integer DEFAULT 100, _timelimit interval DEFAULT '00:00:05'::interval, exitwhenfull boolean DEFAULT true, skipcovered boolean DEFAULT true) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_queryables(_collection_ids text[] DEFAULT NULL::text[]) RETURNS jsonb LANGUAGE plpgsql STABLE AS $function$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.maintain_partition_queries(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false, idxconcurrently boolean DEFAULT false) RETURNS SETOF text LANGUAGE plpgsql AS $function$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %s', q; RETURN NEXT q; END LOOP; RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT false) RETURNS text LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) RETURNS searches LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); BEGIN IF ro THEN updatestats := FALSE; END IF; SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT ro THEN IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; END IF; RETURN search; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_partition_stats(_partition text, istrigger boolean DEFAULT false) RETURNS void LANGUAGE plpgsql STRICT SECURITY DEFINER AS $function$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) RETURNS search_wheres LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN IF ro THEN updatestats := FALSE; END IF; IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF ( sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) ) AND NOT ro THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; IF NOT ro THEN INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; END IF; RETURN sw; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.1.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; SET SEARCH_PATH TO pgstac, public; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; BEGIN FOR rec IN ( WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT * FROM i FULL JOIN q USING (field, partition) WHERE lower(iidx) IS DISTINCT FROM lower(qidx) ) LOOP IF rec.iidx IS NULL THEN IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; ELSIF rec.qidx IS NULL AND dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I;', rec.indexname); ELSIF lower(rec.qidx) != lower(rec.iidx) THEN IF dropindexes THEN RETURN NEXT format('DROP INDEX IF EXISTS %I; %s;', rec.indexname, rec.qidx); ELSE IF idxconcurrently THEN RETURN NEXT replace(rec.qidx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE RETURN NEXT rec.qidx; END IF; END IF; ELSIF rebuildindexes and rec.indexname IS NOT NULL THEN IF idxconcurrently THEN RETURN NEXT format('REINDEX INDEX CONCURRENTLY %I;', rec.indexname); ELSE RETURN NEXT format('REINDEX INDEX %I;', rec.indexname); END IF; END IF; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ) ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints.'; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM create_table_constraints(_partition_name, _constraint_dtrange, _constraint_edtrange); PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(datetime),max(datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); BEGIN IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; BEGIN SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.2-0.8.3.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.items_staging_triggerfunc() RETURNS trigger LANGUAGE plpgsql AS $function$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.2.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %s', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT (content_dehydrate(content)).* FROM newdata ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_ignore; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN WITH staging_formatted AS ( SELECT (content_dehydrate(content)).* FROM newdata ), deletes AS ( DELETE FROM items i USING staging_formatted s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s RETURNING i.id, i.collection ) INSERT INTO items SELECT s.* FROM staging_formatted s ON CONFLICT DO NOTHING; RAISE NOTICE 'Doing the delete. %', clock_timestamp() - ts; DELETE FROM items_staging_upsert; END IF; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN IF ro THEN updatestats := FALSE; END IF; IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF ( sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) ) AND NOT ro THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; IF NOT ro THEN INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; END IF; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); BEGIN IF ro THEN updatestats := FALSE; END IF; SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT ro THEN IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); IF _limit <= number_matched THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset = 0 THEN -- no previous paging links := jsonb_build_array( jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); ELSE links := jsonb_build_array( jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ), jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'context', jsonb_build_object( 'limit', _limit, 'matched', number_matched, 'returned', number_returned ), 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.3-0.8.4.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.3.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %s', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN IF ro THEN updatestats := FALSE; END IF; IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF ( sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) ) AND NOT ro THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; IF NOT ro THEN INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; END IF; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); BEGIN IF ro THEN updatestats := FALSE; END IF; SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT ro THEN IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); IF _limit <= number_matched THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset = 0 THEN -- no previous paging links := jsonb_build_array( jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); ELSE links := jsonb_build_array( jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ), jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'context', jsonb_build_object( 'limit', _limit, 'matched', number_matched, 'returned', number_returned ), 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.4-0.8.5.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, wrapper text DEFAULT NULL::text) RETURNS text LANGUAGE plpgsql STABLE AS $function$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.4.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %s', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN IF ro THEN updatestats := FALSE; END IF; IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF ( sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) ) AND NOT ro THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; IF NOT ro THEN INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; END IF; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); BEGIN IF ro THEN updatestats := FALSE; END IF; SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT ro THEN IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); IF _limit <= number_matched THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset = 0 THEN -- no previous paging links := jsonb_build_array( jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); ELSE links := jsonb_build_array( jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ), jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'context', jsonb_build_object( 'limit', _limit, 'matched', number_matched, 'returned', number_returned ), 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.5-0.9.0.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.search_fromhash(_hash text) RETURNS searches LANGUAGE sql STRICT AS $function$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql STABLE PARALLEL SAFE AS $function$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); IF _limit <= number_matched THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset = 0 THEN -- no previous paging links := jsonb_build_array( jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); ELSE links := jsonb_build_array( jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ), jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, wrapper text DEFAULT NULL::text) RETURNS text LANGUAGE plpgsql STABLE AS $function$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.geometrysearch(geom geometry, queryhash text, fields jsonb DEFAULT NULL::jsonb, _scanlimit integer DEFAULT 10000, _limit integer DEFAULT 100, _timelimit interval DEFAULT '00:00:05'::interval, exitwhenfull boolean DEFAULT true, skipcovered boolean DEFAULT true) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) RETURNS searches LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_rows(_where text DEFAULT 'TRUE'::text, _orderby text DEFAULT 'datetime DESC, id DESC'::text, partitions text[] DEFAULT NULL::text[], _limit integer DEFAULT 10) RETURNS SETOF items LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' AS $function$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) RETURNS search_wheres LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.5.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %s', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN IF ro THEN updatestats := FALSE; END IF; IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF ( sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) ) AND NOT ro THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; IF NOT ro THEN INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; END IF; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); BEGIN IF ro THEN updatestats := FALSE; END IF; SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT ro THEN IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); IF _limit <= number_matched THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset = 0 THEN -- no previous paging links := jsonb_build_array( jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); ELSE links := jsonb_build_array( jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ), jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'context', jsonb_build_object( 'limit', _limit, 'matched', number_matched, 'returned', number_returned ), 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.6-0.9.0.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.search_fromhash(_hash text) RETURNS searches LANGUAGE sql STRICT AS $function$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql STABLE PARALLEL SAFE AS $function$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); IF _limit <= number_matched THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset = 0 THEN -- no previous paging links := jsonb_build_array( jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); ELSE links := jsonb_build_array( jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ), jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, wrapper text DEFAULT NULL::text) RETURNS text LANGUAGE plpgsql STABLE AS $function$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.geometrysearch(geom geometry, queryhash text, fields jsonb DEFAULT NULL::jsonb, _scanlimit integer DEFAULT 10000, _limit integer DEFAULT 100, _timelimit interval DEFAULT '00:00:05'::interval, exitwhenfull boolean DEFAULT true, skipcovered boolean DEFAULT true) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) RETURNS searches LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_rows(_where text DEFAULT 'TRUE'::text, _orderby text DEFAULT 'datetime DESC, id DESC'::text, partitions text[] DEFAULT NULL::text[], _limit integer DEFAULT 10) RETURNS SETOF items LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' AS $function$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) RETURNS search_wheres LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.6-0.9.10.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL drop materialized view if exists "pgstac"."partition_steps"; drop view if exists "pgstac"."partition_sys_meta"; drop materialized view if exists "pgstac"."partitions"; drop view if exists "pgstac"."partitions_view"; drop function if exists "pgstac"."dt_constraint"(coid oid, OUT dt tstzrange, OUT edt tstzrange); set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.get_partition_name(relid regclass) RETURNS text LANGUAGE sql STABLE STRICT AS $function$ SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))]; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_tstz_constraint(reloid oid, colname text) RETURNS tstzrange LANGUAGE plpgsql STABLE STRICT AS $function$ DECLARE expr text := NULL; m text[]; ts_lower timestamptz := NULL; ts_upper timestamptz := NULL; lower_inclusive text := '['; upper_inclusive text := ']'; ts timestamptz; BEGIN SELECT INTO expr string_agg(def, ' AND ') FROM pg_constraint JOIN LATERAL pg_get_constraintdef(oid) AS def ON TRUE WHERE conrelid = reloid AND contype = 'c' AND def LIKE '%' || colname || '%' ; IF expr IS NULL THEN RETURN NULL; END IF; RAISE DEBUG 'Constraint expression for % on %: %', colname, reloid::regclass, expr; -- collect all constraints for the specified column FOR m IN SELECT regexp_matches(expr, '[ (]' || colname || $expr$\s*([<>=]{1,2})\s*'([0-9 :+\-]+)'$expr$, 'g') LOOP ts := m[2]::timestamptz; IF m[1] IN ('>', '>=') THEN IF ts_lower IS NULL OR ts > ts_lower OR (ts = ts_lower AND m[1] = '>') THEN ts_lower := ts; lower_inclusive := CASE WHEN m[1] = '>' THEN '(' ELSE '[' END; END IF; ELSIF m[1] IN ('<', '<=') THEN IF ts_upper IS NULL OR ts < ts_upper OR (ts = ts_upper AND m[1] = '<') THEN ts_upper := ts; upper_inclusive := CASE WHEN m[1] = '<' THEN ')' ELSE ']' END; END IF; END IF; END LOOP; RAISE DEBUG 'Constraint % for %: % %', colname, reloid::regclass, ts_lower, ts_upper; RETURN tstzrange(ts_lower, ts_upper, lower_inclusive || upper_inclusive); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.q_to_tsquery(jinput jsonb) RETURNS tsquery LANGUAGE plpgsql AS $function$ DECLARE input text; processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN IF jsonb_typeof(jinput) = 'string' THEN input := jinput->>0; ELSIF jsonb_typeof(jinput) = 'array' THEN input := array_to_string( array(select jsonb_array_elements_text(jinput)), ' OR ' ); ELSE RAISE EXCEPTION 'Input must be a string or an array of strings.'; END IF; -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- terms separated with spaces are assumed to represent adjacent terms. loop through these -- occurrences and replace them with the adjacency operator (<->) LOOP temp_text := regexp_replace(processed_text, '([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_]+)(?!\s*[&|<>])', '\1 <-> \2', 'g'); IF temp_text = processed_text THEN EXIT; -- No more replacements were made END IF; processed_text := temp_text; END LOOP; -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_fromhash(_hash text) RETURNS searches LANGUAGE sql STRICT AS $function$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $function$ ; CREATE OR REPLACE FUNCTION pgstac.chunker(_where text, OUT s timestamp with time zone, OUT e timestamp with time zone) RETURNS SETOF record LANGUAGE plpgsql AS $function$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_extent(_collection text, runupdate boolean DEFAULT false) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql STABLE PARALLEL SAFE AS $function$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_temporal_extent(id text) RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE SET search_path TO 'pgstac', 'public' AS $function$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.cql2_query(j jsonb, wrapper text DEFAULT NULL::text) RETURNS text LANGUAGE plpgsql STABLE AS $function$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.geometrysearch(geom geometry, queryhash text, fields jsonb DEFAULT NULL::jsonb, _scanlimit integer DEFAULT 10000, _limit integer DEFAULT 100, _timelimit interval DEFAULT '00:00:05'::interval, exitwhenfull boolean DEFAULT true, skipcovered boolean DEFAULT true) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.jsonb_include(j jsonb, f jsonb) RETURNS jsonb LANGUAGE plpgsql IMMUTABLE AS $function$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || ( CASE WHEN j ? 'collection' THEN '["id","collection"]' ELSE '["id"]' END)::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.maintain_partition_queries(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false, idxconcurrently boolean DEFAULT false) RETURNS SETOF text LANGUAGE plpgsql AS $function$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $function$ ; create or replace view "pgstac"."partition_sys_meta" as SELECT partition.partition, replace(replace( CASE WHEN (pg_partition_tree.level = 1) THEN partition_expr.partition_expr ELSE parent_partition_expr.parent_partition_expr END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, pg_partition_tree.level, c.reltuples, c.relhastriggers, partition_dtrange.partition_dtrange, COALESCE(get_tstz_constraint(c.oid, 'datetime'::text), partition_dtrange.partition_dtrange, inf_range.inf_range) AS constraint_dtrange, COALESCE(get_tstz_constraint(c.oid, 'end_datetime'::text), inf_range.inf_range) AS constraint_edtrange FROM ((((((((((pg_partition_tree('items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) LEFT JOIN pg_constraint edt ON (((edt.conrelid = c.oid) AND (edt.contype = 'c'::"char")))) JOIN LATERAL get_partition_name(pg_partition_tree.relid) partition(partition) ON (true)) JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) partition_expr(partition_expr) ON (true)) JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) parent_partition_expr(parent_partition_expr) ON (true)) JOIN LATERAL tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text) inf_range(inf_range) ON (true)) JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range.inf_range) partition_dtrange(partition_dtrange) ON (true)) JOIN LATERAL get_tstz_constraint(c.oid, 'datetime'::text) datetime_constraint(datetime_constraint) ON (true)) JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime'::text) end_datetime_constraint(end_datetime_constraint) ON (true)) WHERE pg_partition_tree.isleaf; create or replace view "pgstac"."partitions_view" as SELECT (parse_ident((pg_partition_tree.relid)::text))[cardinality(parse_ident((pg_partition_tree.relid)::text))] AS partition, replace(replace( CASE WHEN (pg_partition_tree.level = 1) THEN partition_expr.partition_expr ELSE parent_partition_expr.parent_partition_expr END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, pg_partition_tree.level, c.reltuples, c.relhastriggers, partition_dtrange.partition_dtrange, COALESCE(get_tstz_constraint(c.oid, 'datetime'::text), partition_dtrange.partition_dtrange, inf_range.inf_range) AS constraint_dtrange, COALESCE(get_tstz_constraint(c.oid, 'end_datetime'::text), inf_range.inf_range) AS constraint_edtrange, partition_stats.dtrange, partition_stats.edtrange, partition_stats.spatial, partition_stats.last_updated FROM (((((((((((pg_partition_tree('items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) LEFT JOIN pg_constraint edt ON (((edt.conrelid = c.oid) AND (edt.contype = 'c'::"char")))) JOIN LATERAL get_partition_name(pg_partition_tree.relid) partition(partition) ON (true)) JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) partition_expr(partition_expr) ON (true)) JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) parent_partition_expr(parent_partition_expr) ON (true)) JOIN LATERAL tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text) inf_range(inf_range) ON (true)) JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range.inf_range) partition_dtrange(partition_dtrange) ON (true)) JOIN LATERAL get_tstz_constraint(c.oid, 'datetime'::text) datetime_constraint(datetime_constraint) ON (true)) JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime'::text) end_datetime_constraint(end_datetime_constraint) ON (true)) LEFT JOIN partition_stats USING (partition)) WHERE pg_partition_tree.isleaf; CREATE OR REPLACE FUNCTION pgstac.queryable(dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text) RETURNS record LANGUAGE plpgsql STABLE STRICT AS $function$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_query(_search jsonb DEFAULT '{}'::jsonb, updatestats boolean DEFAULT false, _metadata jsonb DEFAULT '{}'::jsonb) RETURNS searches LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_rows(_where text DEFAULT 'TRUE'::text, _orderby text DEFAULT 'datetime DESC, id DESC'::text, partitions text[] DEFAULT NULL::text[], _limit integer DEFAULT 10) RETURNS SETOF items LANGUAGE plpgsql SET search_path TO 'pgstac', 'public' AS $function$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.stac_search_to_where(j jsonb) RETURNS text LANGUAGE plpgsql STABLE AS $function$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_collection_extents() RETURNS void LANGUAGE sql AS $function$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_partition_stats(_partition text, istrigger boolean DEFAULT false) RETURNS void LANGUAGE plpgsql STRICT SECURITY DEFINER AS $function$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) RETURNS search_wheres LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $function$ ; create materialized view "pgstac"."partition_steps" as SELECT partition AS name, date_trunc('month'::text, lower(partition_dtrange)) AS sdate, (date_trunc('month'::text, upper(partition_dtrange)) + '1 mon'::interval) AS edate FROM partitions_view WHERE ((partition_dtrange IS NOT NULL) AND (partition_dtrange <> 'empty'::tstzrange)) ORDER BY dtrange; create materialized view "pgstac"."partitions" as SELECT partition, collection, level, reltuples, relhastriggers, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM partitions_view; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.10'); ================================================ FILE: src/pgstac/migrations/pgstac.0.8.6.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %s', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1->0, args->1->1 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats(inwhere text, updatestats boolean default false, conf jsonb default null) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost float := context_estimated_cost(conf); _estimated_count int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN IF ro THEN updatestats := FALSE; END IF; IF _context = 'off' THEN sw._where := inwhere; return sw; END IF; SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- Update statistics if explicitly set, if statistics do not exist, or statistics ttl has expired IF NOT updatestats THEN RAISE NOTICE 'Checking if update is needed for: % .', inwhere; RAISE NOTICE 'Stats Last Updated: %', sw.statslastupdated; RAISE NOTICE 'TTL: %, Age: %', _stats_ttl, now() - sw.statslastupdated; RAISE NOTICE 'Context: %, Existing Total: %', _context, sw.total_count; IF ( sw.statslastupdated IS NULL OR (now() - sw.statslastupdated) > _stats_ttl OR (context(conf) != 'off' AND sw.total_count IS NULL) ) AND NOT ro THEN updatestats := TRUE; END IF; END IF; sw._where := inwhere; sw.lastused := now(); sw.usecount := coalesce(sw.usecount,0) + 1; IF NOT updatestats THEN UPDATE search_wheres SET lastused = sw.lastused, usecount = sw.usecount WHERE md5(_where) = inwhere_hash RETURNING * INTO sw ; RETURN sw; END IF; -- Use explain to get estimated count/cost and a list of the partitions that would be hit by the query t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE NOTICE 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.statslastupdated := now(); sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); RAISE NOTICE 'ESTIMATED_COUNT: % < %', sw.estimated_count, _estimated_count; RAISE NOTICE 'ESTIMATED_COST: % < %', sw.estimated_cost, _estimated_cost; -- Do a full count of rows if context is set to on or if auto is set and estimates are low enough IF _context = 'on' OR ( _context = 'auto' AND ( sw.estimated_count < _estimated_count AND sw.estimated_cost < _estimated_cost ) ) THEN t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); ELSE sw.total_count := NULL; sw.time_to_count := NULL; END IF; IF NOT ro THEN INSERT INTO search_wheres (_where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, partitions, total_count, time_to_count) SELECT sw._where, sw.lastused, sw.usecount, sw.statslastupdated, sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.partitions, sw.total_count, sw.time_to_count ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = sw.lastused, usecount = sw.usecount, statslastupdated = sw.statslastupdated, estimated_count = sw.estimated_count, estimated_cost = sw.estimated_cost, time_to_estimate = sw.time_to_estimate, total_count = sw.total_count, time_to_count = sw.time_to_count ; END IF; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; _hash text := search_hash(_search, _metadata); doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); BEGIN IF ro THEN updatestats := FALSE; END IF; SELECT * INTO search FROM searches WHERE hash=_hash; search.hash := _hash; -- Calculate the where clause if not already calculated IF search._where IS NULL THEN search._where := stac_search_to_where(_search); ELSE doupdate := TRUE; END IF; -- Calculate the order by clause if not already calculated IF search.orderby IS NULL THEN search.orderby := sort_sqlorderby(_search); ELSE doupdate := TRUE; END IF; PERFORM where_stats(search._where, updatestats, _search->'conf'); IF NOT ro THEN IF NOT doupdate THEN INSERT INTO searches (search, _where, orderby, lastused, usecount, metadata) VALUES (_search, search._where, search.orderby, clock_timestamp(), 1, _metadata) ON CONFLICT (hash) DO NOTHING RETURNING * INTO search; IF FOUND THEN RETURN search; END IF; END IF; UPDATE searches SET lastused=clock_timestamp(), usecount=usecount+1 WHERE hash=( SELECT hash FROM searches WHERE hash=_hash FOR UPDATE SKIP LOCKED ); IF NOT FOUND THEN RAISE NOTICE 'Did not update stats for % due to lock. (This is generally OK)', _search; END IF; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE LOG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE LOG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE LOG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE LOG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE LOG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE LOG 'TOKEN: % %', token_item.id, token_item.collection; RAISE LOG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE LOG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); IF context(_search->'conf') != 'off' THEN context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'matched', total_count, 'returned', coalesce(jsonb_array_length(out_records), 0) )); ELSE context := jsonb_strip_nulls(jsonb_build_object( 'limit', _limit, 'returned', coalesce(jsonb_array_length(out_records), 0) )); END IF; collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'next', next, 'prev', prev, 'context', context ); IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->'context'->>'returned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); IF _limit <= number_matched THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset = 0 THEN -- no previous paging links := jsonb_build_array( jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); ELSE links := jsonb_build_array( jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ), jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'context', jsonb_build_object( 'limit', _limit, 'matched', number_matched, 'returned', number_returned ), 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; SELECT * INTO search FROM searches WHERE hash=queryhash; IF NOT FOUND THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.8.6'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.0-0.9.1.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.collection_extent(_collection text, runupdate boolean DEFAULT false) RETURNS jsonb LANGUAGE plpgsql AS $function$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_temporal_extent(id text) RETURNS jsonb LANGUAGE sql IMMUTABLE PARALLEL SAFE SET search_path TO 'pgstac', 'public' AS $function$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_collection_extents() RETURNS void LANGUAGE sql AS $function$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.0.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %s', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime)::text, max(datetime)::text]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = content || jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', collection_bbox(collections.id) ), 'temporal', jsonb_build_object( 'interval', collection_temporal_extent(collections.id) ) ) ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); IF _limit <= number_matched THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset = 0 THEN -- no previous paging links := jsonb_build_array( jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); ELSE links := jsonb_build_array( jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ), jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'extent', jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.0'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.1-0.9.2.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.q_to_tsquery(input text) RETURNS tsquery LANGUAGE plpgsql AS $function$ DECLARE processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- +term -> & term processed_text := regexp_replace(processed_text, '\+([a-zA-Z0-9_]+)', '& \1', 'g'); -- -term -> ! term processed_text := regexp_replace(processed_text, '\-([a-zA-Z0-9_]+)', '& ! \1', 'g'); -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery(processed_text); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.chunker(_where text, OUT s timestamp with time zone, OUT e timestamp with time zone) RETURNS SETOF record LANGUAGE plpgsql AS $function$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.collection_search(_search jsonb DEFAULT '{}'::jsonb) RETURNS jsonb LANGUAGE plpgsql STABLE PARALLEL SAFE AS $function$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.maintain_partition_queries(part text DEFAULT 'items'::text, dropindexes boolean DEFAULT false, rebuildindexes boolean DEFAULT false, idxconcurrently boolean DEFAULT false) RETURNS SETOF text LANGUAGE plpgsql AS $function$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $function$ ; create or replace view "pgstac"."partition_sys_meta" as SELECT (parse_ident((pg_partition_tree.relid)::text))[cardinality(parse_ident((pg_partition_tree.relid)::text))] AS partition, replace(replace( CASE WHEN (pg_partition_tree.level = 1) THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, pg_partition_tree.level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_edtrange FROM (((pg_partition_tree('items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) LEFT JOIN pg_constraint edt ON (((edt.conrelid = c.oid) AND (edt.contype = 'c'::"char")))) WHERE pg_partition_tree.isleaf; create or replace view "pgstac"."partitions_view" as SELECT (parse_ident((pg_partition_tree.relid)::text))[cardinality(parse_ident((pg_partition_tree.relid)::text))] AS partition, replace(replace( CASE WHEN (pg_partition_tree.level = 1) THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, pg_partition_tree.level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text)) AS constraint_edtrange, partition_stats.dtrange, partition_stats.edtrange, partition_stats.spatial, partition_stats.last_updated FROM ((((pg_partition_tree('items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) LEFT JOIN pg_constraint edt ON (((edt.conrelid = c.oid) AND (edt.contype = 'c'::"char")))) LEFT JOIN partition_stats ON (((parse_ident((pg_partition_tree.relid)::text))[cardinality(parse_ident((pg_partition_tree.relid)::text))] = partition_stats.partition))) WHERE pg_partition_tree.isleaf; CREATE OR REPLACE FUNCTION pgstac.queryable(dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text) RETURNS record LANGUAGE plpgsql STABLE STRICT AS $function$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.stac_search_to_where(j jsonb) RETURNS text LANGUAGE plpgsql STABLE AS $function$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->>'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', content->'properties'->>'title') || to_tsvector('english', content->'properties'->'keywords') ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_partition_stats(_partition text, istrigger boolean DEFAULT false) RETURNS void LANGUAGE plpgsql STRICT SECURITY DEFINER AS $function$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.where_stats(inwhere text, updatestats boolean DEFAULT false, conf jsonb DEFAULT NULL::jsonb) RETURNS search_wheres LANGUAGE plpgsql SECURITY DEFINER AS $function$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.1.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %s', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE VIEW partitions_view AS SELECT relid::text as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN partition_stats ON (relid::text=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); IF _limit <= number_matched THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset = 0 THEN -- no previous paging links := jsonb_build_array( jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); ELSE links := jsonb_build_array( jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ), jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ) ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.1'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.10-0.9.11.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL drop function if exists "pgstac"."search_tohash"(jsonb); set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.get_tstz_constraint(reloid oid, colname text) RETURNS tstzrange LANGUAGE plpgsql STABLE STRICT AS $function$ DECLARE expr text := NULL; m text[]; ts_lower timestamptz := NULL; ts_upper timestamptz := NULL; lower_inclusive text := '['; upper_inclusive text := ']'; ts timestamptz; BEGIN SELECT INTO expr string_agg(def, ' AND ') FROM pg_constraint JOIN LATERAL pg_get_constraintdef(oid) AS def ON TRUE WHERE conrelid = reloid AND contype = 'c' AND def LIKE '%' || colname || '%' ; IF expr IS NULL THEN RETURN NULL; END IF; RAISE DEBUG 'Constraint expression for % on %: %', colname, reloid::regclass, expr; -- collect all constraints for the specified column FOR m IN SELECT regexp_matches(expr, '[ (]' || colname || $expr$\s*([<>=]{1,2})\s*'([0-9 :.+\-]+)'$expr$, 'g') LOOP ts := m[2]::timestamptz; IF m[1] IN ('>', '>=') THEN IF ts_lower IS NULL OR ts > ts_lower OR (ts = ts_lower AND m[1] = '>') THEN ts_lower := ts; lower_inclusive := CASE WHEN m[1] = '>' THEN '(' ELSE '[' END; END IF; ELSIF m[1] IN ('<', '<=') THEN IF ts_upper IS NULL OR ts < ts_upper OR (ts = ts_upper AND m[1] = '<') THEN ts_upper := ts; upper_inclusive := CASE WHEN m[1] = '<' THEN ')' ELSE ']' END; END IF; END IF; END LOOP; RAISE DEBUG 'Constraint % for %: % %', colname, reloid::regclass, ts_lower, ts_upper; RETURN tstzrange(ts_lower, ts_upper, lower_inclusive || upper_inclusive); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.search_hash(jsonb, jsonb) RETURNS text LANGUAGE sql IMMUTABLE PARALLEL SAFE AS $function$ SELECT md5(concat(($1 - '{token,limit,context,includes,excludes}'::text[])::text,$2::text)); $function$ ; CREATE OR REPLACE FUNCTION pgstac.update_partition_stats(_partition text, istrigger boolean DEFAULT false) RETURNS void LANGUAGE plpgsql STRICT SECURITY DEFINER AS $function$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; EXECUTE format('ANALYZE %I;', _partition); extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.11'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.10.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || ( CASE WHEN j ? 'collection' THEN '["id","collection"]' ELSE '["id"]' END)::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION get_tstz_constraint(reloid oid, colname text) RETURNS tstzrange AS $$ DECLARE expr text := NULL; m text[]; ts_lower timestamptz := NULL; ts_upper timestamptz := NULL; lower_inclusive text := '['; upper_inclusive text := ']'; ts timestamptz; BEGIN SELECT INTO expr string_agg(def, ' AND ') FROM pg_constraint JOIN LATERAL pg_get_constraintdef(oid) AS def ON TRUE WHERE conrelid = reloid AND contype = 'c' AND def LIKE '%' || colname || '%' ; IF expr IS NULL THEN RETURN NULL; END IF; RAISE DEBUG 'Constraint expression for % on %: %', colname, reloid::regclass, expr; -- collect all constraints for the specified column FOR m IN SELECT regexp_matches(expr, '[ (]' || colname || $expr$\s*([<>=]{1,2})\s*'([0-9 :+\-]+)'$expr$, 'g') LOOP ts := m[2]::timestamptz; IF m[1] IN ('>', '>=') THEN IF ts_lower IS NULL OR ts > ts_lower OR (ts = ts_lower AND m[1] = '>') THEN ts_lower := ts; lower_inclusive := CASE WHEN m[1] = '>' THEN '(' ELSE '[' END; END IF; ELSIF m[1] IN ('<', '<=') THEN IF ts_upper IS NULL OR ts < ts_upper OR (ts = ts_upper AND m[1] = '<') THEN ts_upper := ts; upper_inclusive := CASE WHEN m[1] = '<' THEN ')' ELSE ']' END; END IF; END IF; END LOOP; RAISE DEBUG 'Constraint % for %: % %', colname, reloid::regclass, ts_lower, ts_upper; RETURN tstzrange(ts_lower, ts_upper, lower_inclusive || upper_inclusive); END; $$ LANGUAGE plpgsql STRICT STABLE; CREATE OR REPLACE FUNCTION get_partition_name(relid regclass) RETURNS text AS $$ SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))]; $$ LANGUAGE SQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT partition, replace( replace( CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, 'FOR VALUES IN (''', '' ), ''')', '' ) AS collection, level, c.reltuples, c.relhastriggers, partition_dtrange, COALESCE( get_tstz_constraint(c.oid, 'datetime'), partition_dtrange, inf_range ) as constraint_dtrange, COALESCE( get_tstz_constraint(c.oid, 'end_datetime'), inf_range ) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace( replace( CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, 'FOR VALUES IN (''', '' ), ''')', '' ) AS collection, level, c.reltuples, c.relhastriggers, partition_dtrange, COALESCE( get_tstz_constraint(c.oid, 'datetime'), partition_dtrange, inf_range ) as constraint_dtrange, COALESCE( get_tstz_constraint(c.oid, 'end_datetime'), inf_range ) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE LEFT JOIN pgstac.partition_stats USING (partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (jinput jsonb) RETURNS tsquery AS $$ DECLARE input text; processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN IF jsonb_typeof(jinput) = 'string' THEN input := jinput->>0; ELSIF jsonb_typeof(jinput) = 'array' THEN input := array_to_string( array(select jsonb_array_elements_text(jinput)), ' OR ' ); ELSE RAISE EXCEPTION 'Input must be a string or an array of strings.'; END IF; -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- terms separated with spaces are assumed to represent adjacent terms. loop through these -- occurrences and replace them with the adjacency operator (<->) LOOP temp_text := regexp_replace(processed_text, '([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_]+)(?!\s*[&|<>])', '\1 <-> \2', 'g'); IF temp_text = processed_text THEN EXIT; -- No more replacements were made END IF; processed_text := temp_text; END LOOP; -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.10'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.11-unreleased.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('unreleased'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.11.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || ( CASE WHEN j ? 'collection' THEN '["id","collection"]' ELSE '["id"]' END)::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION get_tstz_constraint(reloid oid, colname text) RETURNS tstzrange AS $$ DECLARE expr text := NULL; m text[]; ts_lower timestamptz := NULL; ts_upper timestamptz := NULL; lower_inclusive text := '['; upper_inclusive text := ']'; ts timestamptz; BEGIN SELECT INTO expr string_agg(def, ' AND ') FROM pg_constraint JOIN LATERAL pg_get_constraintdef(oid) AS def ON TRUE WHERE conrelid = reloid AND contype = 'c' AND def LIKE '%' || colname || '%' ; IF expr IS NULL THEN RETURN NULL; END IF; RAISE DEBUG 'Constraint expression for % on %: %', colname, reloid::regclass, expr; -- collect all constraints for the specified column FOR m IN SELECT regexp_matches(expr, '[ (]' || colname || $expr$\s*([<>=]{1,2})\s*'([0-9 :.+\-]+)'$expr$, 'g') LOOP ts := m[2]::timestamptz; IF m[1] IN ('>', '>=') THEN IF ts_lower IS NULL OR ts > ts_lower OR (ts = ts_lower AND m[1] = '>') THEN ts_lower := ts; lower_inclusive := CASE WHEN m[1] = '>' THEN '(' ELSE '[' END; END IF; ELSIF m[1] IN ('<', '<=') THEN IF ts_upper IS NULL OR ts < ts_upper OR (ts = ts_upper AND m[1] = '<') THEN ts_upper := ts; upper_inclusive := CASE WHEN m[1] = '<' THEN ')' ELSE ']' END; END IF; END IF; END LOOP; RAISE DEBUG 'Constraint % for %: % %', colname, reloid::regclass, ts_lower, ts_upper; RETURN tstzrange(ts_lower, ts_upper, lower_inclusive || upper_inclusive); END; $$ LANGUAGE plpgsql STRICT STABLE; CREATE OR REPLACE FUNCTION get_partition_name(relid regclass) RETURNS text AS $$ SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))]; $$ LANGUAGE SQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT partition, replace( replace( CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, 'FOR VALUES IN (''', '' ), ''')', '' ) AS collection, level, c.reltuples, c.relhastriggers, partition_dtrange, COALESCE( get_tstz_constraint(c.oid, 'datetime'), partition_dtrange, inf_range ) as constraint_dtrange, COALESCE( get_tstz_constraint(c.oid, 'end_datetime'), inf_range ) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace( replace( CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, 'FOR VALUES IN (''', '' ), ''')', '' ) AS collection, level, c.reltuples, c.relhastriggers, partition_dtrange, COALESCE( get_tstz_constraint(c.oid, 'datetime'), partition_dtrange, inf_range ) as constraint_dtrange, COALESCE( get_tstz_constraint(c.oid, 'end_datetime'), inf_range ) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE LEFT JOIN pgstac.partition_stats USING (partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; EXECUTE format('ANALYZE %I;', _partition); extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (jinput jsonb) RETURNS tsquery AS $$ DECLARE input text; processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN IF jsonb_typeof(jinput) = 'string' THEN input := jinput->>0; ELSIF jsonb_typeof(jinput) = 'array' THEN input := array_to_string( array(select jsonb_array_elements_text(jinput)), ' OR ' ); ELSE RAISE EXCEPTION 'Input must be a string or an array of strings.'; END IF; -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- terms separated with spaces are assumed to represent adjacent terms. loop through these -- occurrences and replace them with the adjacency operator (<->) LOOP temp_text := regexp_replace(processed_text, '([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_]+)(?!\s*[&|<>])', '\1 <-> \2', 'g'); IF temp_text = processed_text THEN EXIT; -- No more replacements were made END IF; processed_text := temp_text; END LOOP; -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(($1 - '{token,limit,context,includes,excludes}'::text[])::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; DROP FUNCTION IF EXISTS search_tohash(jsonb); CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.11'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.2-0.9.3.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.stac_search_to_where(j jsonb) RETURNS text LANGUAGE plpgsql STABLE AS $function$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->>'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.2.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('pgstac.items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN pgstac.partition_stats ON ((parse_ident(relid::text))[cardinality(parse_ident(relid::text))]=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (input text) RETURNS tsquery AS $$ DECLARE processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- +term -> & term processed_text := regexp_replace(processed_text, '\+([a-zA-Z0-9_]+)', '& \1', 'g'); -- -term -> ! term processed_text := regexp_replace(processed_text, '\-([a-zA-Z0-9_]+)', '& ! \1', 'g'); -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery(processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->>'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', content->'properties'->>'title') || to_tsvector('english', content->'properties'->'keywords') ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.2'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.3-0.9.4.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.3.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('pgstac.items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN pgstac.partition_stats ON ((parse_ident(relid::text))[cardinality(parse_ident(relid::text))]=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (input text) RETURNS tsquery AS $$ DECLARE processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- +term -> & term processed_text := regexp_replace(processed_text, '\+([a-zA-Z0-9_]+)', '& \1', 'g'); -- -term -> ! term processed_text := regexp_replace(processed_text, '\-([a-zA-Z0-9_]+)', '& ! \1', 'g'); -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery(processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->>'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.3'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.4-0.9.5.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.4.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('pgstac.items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN pgstac.partition_stats ON ((parse_ident(relid::text))[cardinality(parse_ident(relid::text))]=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (input text) RETURNS tsquery AS $$ DECLARE processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- +term -> & term processed_text := regexp_replace(processed_text, '\+([a-zA-Z0-9_]+)', '& \1', 'g'); -- -term -> ! term processed_text := regexp_replace(processed_text, '\-([a-zA-Z0-9_]+)', '& ! \1', 'g'); -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery(processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->>'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.4'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.5-0.9.6.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.6'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.5.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('pgstac.items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN pgstac.partition_stats ON ((parse_ident(relid::text))[cardinality(parse_ident(relid::text))]=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (input text) RETURNS tsquery AS $$ DECLARE processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- +term -> & term processed_text := regexp_replace(processed_text, '\+([a-zA-Z0-9_]+)', '& \1', 'g'); -- -term -> ! term processed_text := regexp_replace(processed_text, '\-([a-zA-Z0-9_]+)', '& ! \1', 'g'); -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery(processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->>'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.5'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.6-0.9.7.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.q_to_tsquery(input text) RETURNS tsquery LANGUAGE plpgsql AS $function$ DECLARE processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.7'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.6.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('pgstac.items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN pgstac.partition_stats ON ((parse_ident(relid::text))[cardinality(parse_ident(relid::text))]=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (input text) RETURNS tsquery AS $$ DECLARE processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- +term -> & term processed_text := regexp_replace(processed_text, '\+([a-zA-Z0-9_]+)', '& \1', 'g'); -- -term -> ! term processed_text := regexp_replace(processed_text, '\-([a-zA-Z0-9_]+)', '& ! \1', 'g'); -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery(processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->>'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.6'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.7-0.9.8.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL drop function if exists "pgstac"."q_to_tsquery"(input text); set check_function_bodies = off; DROP FUNCTION IF EXISTS pgstac.q_to_tsquery(text); CREATE OR REPLACE FUNCTION pgstac.q_to_tsquery(jinput jsonb) RETURNS tsquery LANGUAGE plpgsql AS $function$ DECLARE input text; processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN IF jsonb_typeof(jinput) = 'string' THEN input := jinput->>0; ELSIF jsonb_typeof(jinput) = 'array' THEN input := array_to_string( array(select jsonb_array_elements_text(jinput)), ' OR ' ); ELSE RAISE EXCEPTION 'Input must be a string or an array of strings.'; END IF; -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.stac_search_to_where(j jsonb) RETURNS text LANGUAGE plpgsql STABLE AS $function$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.8'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.7.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('pgstac.items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN pgstac.partition_stats ON ((parse_ident(relid::text))[cardinality(parse_ident(relid::text))]=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (input text) RETURNS tsquery AS $$ DECLARE processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->>'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.7'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.8-0.9.9.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.jsonb_include(j jsonb, f jsonb) RETURNS jsonb LANGUAGE plpgsql IMMUTABLE AS $function$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || ( CASE WHEN j ? 'collection' THEN '["id","collection"]' ELSE '["id"]' END)::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $function$ ; CREATE OR REPLACE FUNCTION pgstac.q_to_tsquery(jinput jsonb) RETURNS tsquery LANGUAGE plpgsql AS $function$ DECLARE input text; processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN IF jsonb_typeof(jinput) = 'string' THEN input := jinput->>0; ELSIF jsonb_typeof(jinput) = 'array' THEN input := array_to_string( array(select jsonb_array_elements_text(jinput)), ' OR ' ); ELSE RAISE EXCEPTION 'Input must be a string or an array of strings.'; END IF; -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- terms separated with spaces are assumed to represent adjacent terms. loop through these -- occurrences and replace them with the adjacency operator (<->) LOOP temp_text := regexp_replace(processed_text, '([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_]+)(?!\s*[&|<>])', '\1 <-> \2', 'g'); IF temp_text = processed_text THEN EXIT; -- No more replacements were made END IF; processed_text := temp_text; END LOOP; -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $function$ ; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.9'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.8.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || '["id","collection"]'::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('pgstac.items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN pgstac.partition_stats ON ((parse_ident(relid::text))[cardinality(parse_ident(relid::text))]=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (jinput jsonb) RETURNS tsquery AS $$ DECLARE input text; processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN IF jsonb_typeof(jinput) = 'string' THEN input := jinput->>0; ELSIF jsonb_typeof(jinput) = 'array' THEN input := array_to_string( array(select jsonb_array_elements_text(jinput)), ' OR ' ); ELSE RAISE EXCEPTION 'Input must be a string or an array of strings.'; END IF; -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.8'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.9-0.9.10.sql ================================================ SET client_min_messages TO WARNING; SET SEARCH_PATH to pgstac, public; RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; -- BEGIN migra calculated SQL drop materialized view if exists "pgstac"."partition_steps"; drop view if exists "pgstac"."partition_sys_meta"; drop materialized view if exists "pgstac"."partitions"; drop view if exists "pgstac"."partitions_view"; drop function if exists "pgstac"."dt_constraint"(coid oid, OUT dt tstzrange, OUT edt tstzrange); set check_function_bodies = off; CREATE OR REPLACE FUNCTION pgstac.get_partition_name(relid regclass) RETURNS text LANGUAGE sql STABLE STRICT AS $function$ SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))]; $function$ ; CREATE OR REPLACE FUNCTION pgstac.get_tstz_constraint(reloid oid, colname text) RETURNS tstzrange LANGUAGE plpgsql STABLE STRICT AS $function$ DECLARE expr text := NULL; m text[]; ts_lower timestamptz := NULL; ts_upper timestamptz := NULL; lower_inclusive text := '['; upper_inclusive text := ']'; ts timestamptz; BEGIN SELECT INTO expr string_agg(def, ' AND ') FROM pg_constraint JOIN LATERAL pg_get_constraintdef(oid) AS def ON TRUE WHERE conrelid = reloid AND contype = 'c' AND def LIKE '%' || colname || '%' ; IF expr IS NULL THEN RETURN NULL; END IF; RAISE DEBUG 'Constraint expression for % on %: %', colname, reloid::regclass, expr; -- collect all constraints for the specified column FOR m IN SELECT regexp_matches(expr, '[ (]' || colname || $expr$\s*([<>=]{1,2})\s*'([0-9 :+\-]+)'$expr$, 'g') LOOP ts := m[2]::timestamptz; IF m[1] IN ('>', '>=') THEN IF ts_lower IS NULL OR ts > ts_lower OR (ts = ts_lower AND m[1] = '>') THEN ts_lower := ts; lower_inclusive := CASE WHEN m[1] = '>' THEN '(' ELSE '[' END; END IF; ELSIF m[1] IN ('<', '<=') THEN IF ts_upper IS NULL OR ts < ts_upper OR (ts = ts_upper AND m[1] = '<') THEN ts_upper := ts; upper_inclusive := CASE WHEN m[1] = '<' THEN ')' ELSE ']' END; END IF; END IF; END LOOP; RAISE DEBUG 'Constraint % for %: % %', colname, reloid::regclass, ts_lower, ts_upper; RETURN tstzrange(ts_lower, ts_upper, lower_inclusive || upper_inclusive); END; $function$ ; create or replace view "pgstac"."partition_sys_meta" as SELECT partition.partition, replace(replace( CASE WHEN (pg_partition_tree.level = 1) THEN partition_expr.partition_expr ELSE parent_partition_expr.parent_partition_expr END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, pg_partition_tree.level, c.reltuples, c.relhastriggers, partition_dtrange.partition_dtrange, COALESCE(get_tstz_constraint(c.oid, 'datetime'::text), partition_dtrange.partition_dtrange, inf_range.inf_range) AS constraint_dtrange, COALESCE(get_tstz_constraint(c.oid, 'end_datetime'::text), inf_range.inf_range) AS constraint_edtrange FROM ((((((((((pg_partition_tree('items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) LEFT JOIN pg_constraint edt ON (((edt.conrelid = c.oid) AND (edt.contype = 'c'::"char")))) JOIN LATERAL get_partition_name(pg_partition_tree.relid) partition(partition) ON (true)) JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) partition_expr(partition_expr) ON (true)) JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) parent_partition_expr(parent_partition_expr) ON (true)) JOIN LATERAL tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text) inf_range(inf_range) ON (true)) JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range.inf_range) partition_dtrange(partition_dtrange) ON (true)) JOIN LATERAL get_tstz_constraint(c.oid, 'datetime'::text) datetime_constraint(datetime_constraint) ON (true)) JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime'::text) end_datetime_constraint(end_datetime_constraint) ON (true)) WHERE pg_partition_tree.isleaf; create or replace view "pgstac"."partitions_view" as SELECT (parse_ident((pg_partition_tree.relid)::text))[cardinality(parse_ident((pg_partition_tree.relid)::text))] AS partition, replace(replace( CASE WHEN (pg_partition_tree.level = 1) THEN partition_expr.partition_expr ELSE parent_partition_expr.parent_partition_expr END, 'FOR VALUES IN ('''::text, ''::text), ''')'::text, ''::text) AS collection, pg_partition_tree.level, c.reltuples, c.relhastriggers, partition_dtrange.partition_dtrange, COALESCE(get_tstz_constraint(c.oid, 'datetime'::text), partition_dtrange.partition_dtrange, inf_range.inf_range) AS constraint_dtrange, COALESCE(get_tstz_constraint(c.oid, 'end_datetime'::text), inf_range.inf_range) AS constraint_edtrange, partition_stats.dtrange, partition_stats.edtrange, partition_stats.spatial, partition_stats.last_updated FROM (((((((((((pg_partition_tree('items'::regclass) pg_partition_tree(relid, parentrelid, isleaf, level) JOIN pg_class c ON (((pg_partition_tree.relid)::oid = c.oid))) JOIN pg_class parent ON ((((pg_partition_tree.parentrelid)::oid = parent.oid) AND pg_partition_tree.isleaf))) LEFT JOIN pg_constraint edt ON (((edt.conrelid = c.oid) AND (edt.contype = 'c'::"char")))) JOIN LATERAL get_partition_name(pg_partition_tree.relid) partition(partition) ON (true)) JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) partition_expr(partition_expr) ON (true)) JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) parent_partition_expr(parent_partition_expr) ON (true)) JOIN LATERAL tstzrange('-infinity'::timestamp with time zone, 'infinity'::timestamp with time zone, '[]'::text) inf_range(inf_range) ON (true)) JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range.inf_range) partition_dtrange(partition_dtrange) ON (true)) JOIN LATERAL get_tstz_constraint(c.oid, 'datetime'::text) datetime_constraint(datetime_constraint) ON (true)) JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime'::text) end_datetime_constraint(end_datetime_constraint) ON (true)) LEFT JOIN partition_stats USING (partition)) WHERE pg_partition_tree.isleaf; create materialized view "pgstac"."partition_steps" as SELECT partition AS name, date_trunc('month'::text, lower(partition_dtrange)) AS sdate, (date_trunc('month'::text, upper(partition_dtrange)) + '1 mon'::interval) AS edate FROM partitions_view WHERE ((partition_dtrange IS NOT NULL) AND (partition_dtrange <> 'empty'::tstzrange)) ORDER BY dtrange; create materialized view "pgstac"."partitions" as SELECT partition, collection, level, reltuples, relhastriggers, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM partitions_view; -- END migra calculated SQL DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.10'); ================================================ FILE: src/pgstac/migrations/pgstac.0.9.9.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || ( CASE WHEN j ? 'collection' THEN '["id","collection"]' ELSE '["id"]' END)::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION dt_constraint(coid oid, OUT dt tstzrange, OUT edt tstzrange) RETURNS RECORD AS $$ DECLARE expr text := pg_get_constraintdef(coid); matches timestamptz[]; BEGIN IF expr LIKE '%NULL%' THEN dt := tstzrange(null::timestamptz, null::timestamptz); edt := tstzrange(null::timestamptz, null::timestamptz); RETURN; END IF; WITH f AS (SELECT (regexp_matches(expr, E'([0-9]{4}-[0-1][0-9]-[0-3][0-9] [0-2][0-9]:[0-5][0-9]:[0-5][0-9]\.?[0-9]*)', 'g'))[1] f) SELECT array_agg(f::timestamptz) INTO matches FROM f; IF cardinality(matches) = 4 THEN dt := tstzrange(matches[1], matches[2],'[]'); edt := tstzrange(matches[3], matches[4], '[]'); RETURN; ELSIF cardinality(matches) = 2 THEN edt := tstzrange(matches[1], matches[2],'[]'); RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((dt_constraint(edt.oid)).dt, constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace(replace(CASE WHEN level = 1 THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')','') AS collection, level, c.reltuples, c.relhastriggers, COALESCE(pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as partition_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).dt, pgstac.constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), tstzrange('-infinity', 'infinity','[]')) as constraint_dtrange, COALESCE((pgstac.dt_constraint(edt.oid)).edt, tstzrange('-infinity', 'infinity','[]')) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('pgstac.items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') LEFT JOIN pgstac.partition_stats ON ((parse_ident(relid::text))[cardinality(parse_ident(relid::text))]=partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; END IF; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (jinput jsonb) RETURNS tsquery AS $$ DECLARE input text; processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN IF jsonb_typeof(jinput) = 'string' THEN input := jinput->>0; ELSIF jsonb_typeof(jinput) = 'array' THEN input := array_to_string( array(select jsonb_array_elements_text(jinput)), ' OR ' ); ELSE RAISE EXCEPTION 'Input must be a string or an array of strings.'; END IF; -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- terms separated with spaces are assumed to represent adjacent terms. loop through these -- occurrences and replace them with the adjacency operator (<->) LOOP temp_text := regexp_replace(processed_text, '([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_]+)(?!\s*[&|<>])', '\1 <-> \2', 'g'); IF temp_text = processed_text THEN EXIT; -- No more replacements were made END IF; processed_text := temp_text; END LOOP; -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_tohash(jsonb) RETURNS jsonb AS $$ SELECT $1 - '{token,limit,context,includes,excludes}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(search_tohash($1)::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('0.9.9'); ================================================ FILE: src/pgstac/migrations/pgstac.unreleased.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || ( CASE WHEN j ? 'collection' THEN '["id","collection"]' ELSE '["id"]' END)::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION get_tstz_constraint(reloid oid, colname text) RETURNS tstzrange AS $$ DECLARE expr text := NULL; m text[]; ts_lower timestamptz := NULL; ts_upper timestamptz := NULL; lower_inclusive text := '['; upper_inclusive text := ']'; ts timestamptz; BEGIN SELECT INTO expr string_agg(def, ' AND ') FROM pg_constraint JOIN LATERAL pg_get_constraintdef(oid) AS def ON TRUE WHERE conrelid = reloid AND contype = 'c' AND def LIKE '%' || colname || '%' ; IF expr IS NULL THEN RETURN NULL; END IF; RAISE DEBUG 'Constraint expression for % on %: %', colname, reloid::regclass, expr; -- collect all constraints for the specified column FOR m IN SELECT regexp_matches(expr, '[ (]' || colname || $expr$\s*([<>=]{1,2})\s*'([0-9 :.+\-]+)'$expr$, 'g') LOOP ts := m[2]::timestamptz; IF m[1] IN ('>', '>=') THEN IF ts_lower IS NULL OR ts > ts_lower OR (ts = ts_lower AND m[1] = '>') THEN ts_lower := ts; lower_inclusive := CASE WHEN m[1] = '>' THEN '(' ELSE '[' END; END IF; ELSIF m[1] IN ('<', '<=') THEN IF ts_upper IS NULL OR ts < ts_upper OR (ts = ts_upper AND m[1] = '<') THEN ts_upper := ts; upper_inclusive := CASE WHEN m[1] = '<' THEN ')' ELSE ']' END; END IF; END IF; END LOOP; RAISE DEBUG 'Constraint % for %: % %', colname, reloid::regclass, ts_lower, ts_upper; RETURN tstzrange(ts_lower, ts_upper, lower_inclusive || upper_inclusive); END; $$ LANGUAGE plpgsql STRICT STABLE; CREATE OR REPLACE FUNCTION get_partition_name(relid regclass) RETURNS text AS $$ SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))]; $$ LANGUAGE SQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT partition, replace( replace( CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, 'FOR VALUES IN (''', '' ), ''')', '' ) AS collection, level, c.reltuples, c.relhastriggers, partition_dtrange, COALESCE( get_tstz_constraint(c.oid, 'datetime'), partition_dtrange, inf_range ) as constraint_dtrange, COALESCE( get_tstz_constraint(c.oid, 'end_datetime'), inf_range ) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace( replace( CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, 'FOR VALUES IN (''', '' ), ''')', '' ) AS collection, level, c.reltuples, c.relhastriggers, partition_dtrange, COALESCE( get_tstz_constraint(c.oid, 'datetime'), partition_dtrange, inf_range ) as constraint_dtrange, COALESCE( get_tstz_constraint(c.oid, 'end_datetime'), inf_range ) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE LEFT JOIN pgstac.partition_stats USING (partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; EXECUTE format('ANALYZE %I;', _partition); extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (jinput jsonb) RETURNS tsquery AS $$ DECLARE input text; processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN IF jsonb_typeof(jinput) = 'string' THEN input := jinput->>0; ELSIF jsonb_typeof(jinput) = 'array' THEN input := array_to_string( array(select jsonb_array_elements_text(jinput)), ' OR ' ); ELSE RAISE EXCEPTION 'Input must be a string or an array of strings.'; END IF; -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- terms separated with spaces are assumed to represent adjacent terms. loop through these -- occurrences and replace them with the adjacency operator (<->) LOOP temp_text := regexp_replace(processed_text, '([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_]+)(?!\s*[&|<>])', '\1 <-> \2', 'g'); IF temp_text = processed_text THEN EXIT; -- No more replacements were made END IF; processed_text := temp_text; END LOOP; -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(($1 - '{token,limit,context,includes,excludes}'::text[])::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; DROP FUNCTION IF EXISTS search_tohash(jsonb); CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('unreleased'); ================================================ FILE: src/pgstac/pgstac.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || ( CASE WHEN j ? 'collection' THEN '["id","collection"]' ELSE '["id"]' END)::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION get_tstz_constraint(reloid oid, colname text) RETURNS tstzrange AS $$ DECLARE expr text := NULL; m text[]; ts_lower timestamptz := NULL; ts_upper timestamptz := NULL; lower_inclusive text := '['; upper_inclusive text := ']'; ts timestamptz; BEGIN SELECT INTO expr string_agg(def, ' AND ') FROM pg_constraint JOIN LATERAL pg_get_constraintdef(oid) AS def ON TRUE WHERE conrelid = reloid AND contype = 'c' AND def LIKE '%' || colname || '%' ; IF expr IS NULL THEN RETURN NULL; END IF; RAISE DEBUG 'Constraint expression for % on %: %', colname, reloid::regclass, expr; -- collect all constraints for the specified column FOR m IN SELECT regexp_matches(expr, '[ (]' || colname || $expr$\s*([<>=]{1,2})\s*'([0-9 :.+\-]+)'$expr$, 'g') LOOP ts := m[2]::timestamptz; IF m[1] IN ('>', '>=') THEN IF ts_lower IS NULL OR ts > ts_lower OR (ts = ts_lower AND m[1] = '>') THEN ts_lower := ts; lower_inclusive := CASE WHEN m[1] = '>' THEN '(' ELSE '[' END; END IF; ELSIF m[1] IN ('<', '<=') THEN IF ts_upper IS NULL OR ts < ts_upper OR (ts = ts_upper AND m[1] = '<') THEN ts_upper := ts; upper_inclusive := CASE WHEN m[1] = '<' THEN ')' ELSE ']' END; END IF; END IF; END LOOP; RAISE DEBUG 'Constraint % for %: % %', colname, reloid::regclass, ts_lower, ts_upper; RETURN tstzrange(ts_lower, ts_upper, lower_inclusive || upper_inclusive); END; $$ LANGUAGE plpgsql STRICT STABLE; CREATE OR REPLACE FUNCTION get_partition_name(relid regclass) RETURNS text AS $$ SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))]; $$ LANGUAGE SQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT partition, replace( replace( CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, 'FOR VALUES IN (''', '' ), ''')', '' ) AS collection, level, c.reltuples, c.relhastriggers, partition_dtrange, COALESCE( get_tstz_constraint(c.oid, 'datetime'), partition_dtrange, inf_range ) as constraint_dtrange, COALESCE( get_tstz_constraint(c.oid, 'end_datetime'), inf_range ) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace( replace( CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, 'FOR VALUES IN (''', '' ), ''')', '' ) AS collection, level, c.reltuples, c.relhastriggers, partition_dtrange, COALESCE( get_tstz_constraint(c.oid, 'datetime'), partition_dtrange, inf_range ) as constraint_dtrange, COALESCE( get_tstz_constraint(c.oid, 'end_datetime'), inf_range ) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE LEFT JOIN pgstac.partition_stats USING (partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; EXECUTE format('ANALYZE %I;', _partition); extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (jinput jsonb) RETURNS tsquery AS $$ DECLARE input text; processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN IF jsonb_typeof(jinput) = 'string' THEN input := jinput->>0; ELSIF jsonb_typeof(jinput) = 'array' THEN input := array_to_string( array(select jsonb_array_elements_text(jinput)), ' OR ' ); ELSE RAISE EXCEPTION 'Input must be a string or an array of strings.'; END IF; -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- terms separated with spaces are assumed to represent adjacent terms. loop through these -- occurrences and replace them with the adjacency operator (<->) LOOP temp_text := regexp_replace(processed_text, '([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_]+)(?!\s*[&|<>])', '\1 <-> \2', 'g'); IF temp_text = processed_text THEN EXIT; -- No more replacements were made END IF; processed_text := temp_text; END LOOP; -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(($1 - '{token,limit,context,includes,excludes}'::text[])::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; DROP FUNCTION IF EXISTS search_tohash(jsonb); CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; SELECT set_version('unreleased'); ================================================ FILE: src/pgstac/sql/000_idempotent_pre.sql ================================================ RESET ROLE; DO $$ DECLARE BEGIN IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='postgis') THEN CREATE EXTENSION IF NOT EXISTS postgis; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='btree_gist') THEN CREATE EXTENSION IF NOT EXISTS btree_gist; END IF; IF NOT EXISTS (SELECT 1 FROM pg_extension WHERE extname='unaccent') THEN CREATE EXTENSION IF NOT EXISTS unaccent; END IF; END; $$ LANGUAGE PLPGSQL; DO $$ BEGIN CREATE ROLE pgstac_admin; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_read; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN CREATE ROLE pgstac_ingest; EXCEPTION WHEN duplicate_object THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; GRANT pgstac_admin TO current_user; -- Function to make sure pgstac_admin is the owner of items CREATE OR REPLACE FUNCTION pgstac_admin_owns() RETURNS VOID AS $$ DECLARE f RECORD; BEGIN FOR f IN ( SELECT concat( oid::regproc::text, '(', coalesce(pg_get_function_identity_arguments(oid),''), ')' ) AS name, CASE prokind WHEN 'f' THEN 'FUNCTION' WHEN 'p' THEN 'PROCEDURE' WHEN 'a' THEN 'AGGREGATE' END as typ FROM pg_proc WHERE pronamespace=to_regnamespace('pgstac') AND proowner != to_regrole('pgstac_admin') AND proname NOT LIKE 'pg_stat%' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; FOR f IN ( SELECT oid::regclass::text as name, CASE relkind WHEN 'i' THEN 'INDEX' WHEN 'I' THEN 'INDEX' WHEN 'p' THEN 'TABLE' WHEN 'r' THEN 'TABLE' WHEN 'v' THEN 'VIEW' WHEN 'S' THEN 'SEQUENCE' ELSE NULL END as typ FROM pg_class WHERE relnamespace=to_regnamespace('pgstac') and relowner != to_regrole('pgstac_admin') AND relkind IN ('r','p','v','S') AND relname NOT LIKE 'pg_stat' ) LOOP BEGIN EXECUTE format('ALTER %s %s OWNER TO pgstac_admin;', f.typ, f.name); EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; SELECT pgstac_admin_owns(); CREATE SCHEMA IF NOT EXISTS pgstac AUTHORIZATION pgstac_admin; GRANT ALL ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_admin; GRANT ALL ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_admin; ALTER ROLE pgstac_admin SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_read SET SEARCH_PATH TO pgstac, public; ALTER ROLE pgstac_ingest SET SEARCH_PATH TO pgstac, public; GRANT USAGE ON SCHEMA pgstac to pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; GRANT pgstac_read TO pgstac_ingest; GRANT ALL ON SCHEMA pgstac TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; SET ROLE pgstac_admin; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_admin IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET ROLE pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT SELECT ON TABLES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT USAGE ON TYPES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON SEQUENCES TO pgstac_read; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON TABLES TO pgstac_ingest; ALTER DEFAULT PRIVILEGES FOR ROLE pgstac_ingest IN SCHEMA pgstac GRANT ALL ON FUNCTIONS TO pgstac_ingest; RESET ROLE; SET SEARCH_PATH TO pgstac, public; SET ROLE pgstac_admin; DO $$ BEGIN DROP FUNCTION IF EXISTS analyze_items; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN DROP FUNCTION IF EXISTS validate_constraints; EXCEPTION WHEN others THEN RAISE NOTICE '%, skipping', SQLERRM USING ERRCODE = SQLSTATE; END $$; -- Install these idempotently as migrations do not put them before trying to modify the collections table CREATE OR REPLACE FUNCTION collection_geom(content jsonb) RETURNS geometry AS $$ WITH box AS (SELECT content->'extent'->'spatial'->'bbox'->0 as box) SELECT st_makeenvelope( (box->>0)::float, (box->>1)::float, (box->>2)::float, (box->>3)::float, 4326 ) FROM box; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_datetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>0) IS NULL THEN '-infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>0)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION collection_enddatetime(content jsonb) RETURNS timestamptz AS $$ SELECT CASE WHEN (content->'extent'->'temporal'->'interval'->0->>1) IS NULL THEN 'infinity'::timestamptz ELSE (content->'extent'->'temporal'->'interval'->0->>1)::timestamptz END ; $$ LANGUAGE SQL IMMUTABLE STRICT; ================================================ FILE: src/pgstac/sql/001_core.sql ================================================ CREATE TABLE IF NOT EXISTS migrations ( version text PRIMARY KEY, datetime timestamptz DEFAULT clock_timestamp() NOT NULL ); CREATE OR REPLACE FUNCTION get_version() RETURNS text AS $$ SELECT version FROM pgstac.migrations ORDER BY datetime DESC, version DESC LIMIT 1; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION set_version(text) RETURNS text AS $$ INSERT INTO pgstac.migrations (version) VALUES ($1) ON CONFLICT DO NOTHING RETURNING version; $$ LANGUAGE SQL; CREATE TABLE IF NOT EXISTS pgstac_settings ( name text PRIMARY KEY, value text NOT NULL ); CREATE OR REPLACE FUNCTION table_empty(text) RETURNS boolean AS $$ DECLARE retval boolean; BEGIN EXECUTE format($q$ SELECT NOT EXISTS (SELECT 1 FROM %I LIMIT 1) $q$, $1 ) INTO retval; RETURN retval; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_setting(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),'') ); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_setting_bool(IN _setting text, IN conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT COALESCE( nullif(conf->>_setting, ''), nullif(current_setting(concat('pgstac.',_setting), TRUE),''), nullif((SELECT value FROM pgstac.pgstac_settings WHERE name=_setting),''), 'FALSE' )::boolean; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION base_url(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT COALESCE(pgstac.get_setting('base_url', conf), '.'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION additional_properties() RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('additional_properties'); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION readonly(conf jsonb DEFAULT NULL) RETURNS boolean AS $$ SELECT pgstac.get_setting_bool('readonly', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context(conf jsonb DEFAULT NULL) RETURNS text AS $$ SELECT pgstac.get_setting('context', conf); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION context_estimated_count(conf jsonb DEFAULT NULL) RETURNS int AS $$ SELECT pgstac.get_setting('context_estimated_count', conf)::int; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_estimated_cost(); CREATE OR REPLACE FUNCTION context_estimated_cost(conf jsonb DEFAULT NULL) RETURNS float AS $$ SELECT pgstac.get_setting('context_estimated_cost', conf)::float; $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS context_stats_ttl(); CREATE OR REPLACE FUNCTION context_stats_ttl(conf jsonb DEFAULT NULL) RETURNS interval AS $$ SELECT pgstac.get_setting('context_stats_ttl', conf)::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION t2s(text) RETURNS text AS $$ SELECT extract(epoch FROM $1::interval)::text || ' s'; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION age_ms(a timestamptz, b timestamptz DEFAULT clock_timestamp()) RETURNS float AS $$ SELECT abs(extract(epoch from age(a,b)) * 1000); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION queue_timeout() RETURNS interval AS $$ SELECT t2s(coalesce( get_setting('queue_timeout'), '1h' ))::interval; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION notice(VARIADIC text[]) RETURNS boolean AS $$ DECLARE debug boolean := current_setting('pgstac.debug', true); BEGIN IF debug THEN RAISE NOTICE 'NOTICE FROM FUNC: % >>>>> %', concat_ws(' | ', $1), clock_timestamp(); RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION empty_arr(ANYARRAY) RETURNS BOOLEAN AS $$ SELECT CASE WHEN $1 IS NULL THEN TRUE WHEN cardinality($1)<1 THEN TRUE ELSE FALSE END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_intersection(_a ANYARRAY, _b ANYARRAY) RETURNS ANYARRAY AS $$ SELECT ARRAY ( SELECT unnest(_a) INTERSECT SELECT UNNEST(_b) ); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION array_map_ident(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_ident(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_map_literal(_a text[]) RETURNS text[] AS $$ SELECT array_agg(quote_literal(v)) FROM unnest(_a) v; $$ LANGUAGE sql IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION array_reverse(anyarray) RETURNS anyarray AS $$ SELECT ARRAY( SELECT $1[i] FROM generate_subscripts($1,1) AS s(i) ORDER BY i DESC ); $$ LANGUAGE SQL STRICT IMMUTABLE; DROP TABLE IF EXISTS query_queue; CREATE TABLE query_queue ( query text PRIMARY KEY, added timestamptz DEFAULT now() ); DROP TABLE IF EXISTS query_queue_history; CREATE TABLE query_queue_history( query text, added timestamptz NOT NULL, finished timestamptz NOT NULL DEFAULT now(), error text ); CREATE OR REPLACE PROCEDURE run_queued_queries() AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN EXIT; END IF; cnt := cnt + 1; BEGIN RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_queued_queries_intransaction() RETURNS int AS $$ DECLARE qitem query_queue%ROWTYPE; timeout_ts timestamptz; error text; cnt int := 0; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP DELETE FROM query_queue WHERE query = (SELECT query FROM query_queue ORDER BY added DESC LIMIT 1 FOR UPDATE SKIP LOCKED) RETURNING * INTO qitem; IF NOT FOUND THEN RETURN cnt; END IF; cnt := cnt + 1; BEGIN qitem.query := regexp_replace(qitem.query, 'CONCURRENTLY', ''); RAISE NOTICE 'RUNNING QUERY: %', qitem.query; EXECUTE qitem.query; EXCEPTION WHEN others THEN error := format('%s | %s', SQLERRM, SQLSTATE); RAISE WARNING '%', error; END; INSERT INTO query_queue_history (query, added, finished, error) VALUES (qitem.query, qitem.added, clock_timestamp(), error); END LOOP; RETURN cnt; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION run_or_queue(query text) RETURNS VOID AS $$ DECLARE use_queue text := COALESCE(get_setting('use_queue'), 'FALSE')::boolean; BEGIN IF get_setting_bool('debug') THEN RAISE NOTICE '%', query; END IF; IF use_queue THEN INSERT INTO query_queue (query) VALUES (query) ON CONFLICT DO NOTHING; ELSE EXECUTE query; END IF; RETURN; END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS check_pgstac_settings; CREATE OR REPLACE FUNCTION check_pgstac_settings(_sysmem text DEFAULT NULL) RETURNS VOID AS $$ DECLARE settingval text; sysmem bigint := pg_size_bytes(_sysmem); effective_cache_size bigint := pg_size_bytes(current_setting('effective_cache_size', TRUE)); shared_buffers bigint := pg_size_bytes(current_setting('shared_buffers', TRUE)); work_mem bigint := pg_size_bytes(current_setting('work_mem', TRUE)); max_connections int := current_setting('max_connections', TRUE); maintenance_work_mem bigint := pg_size_bytes(current_setting('maintenance_work_mem', TRUE)); seq_page_cost float := current_setting('seq_page_cost', TRUE); random_page_cost float := current_setting('random_page_cost', TRUE); temp_buffers bigint := pg_size_bytes(current_setting('temp_buffers', TRUE)); r record; BEGIN IF _sysmem IS NULL THEN RAISE NOTICE 'Call function with the size of your system memory `SELECT check_pgstac_settings(''4GB'')` to get pg system setting recommendations.'; ELSE IF effective_cache_size < (sysmem * 0.5) THEN RAISE WARNING 'effective_cache_size of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSIF effective_cache_size > (sysmem * 0.75) THEN RAISE WARNING 'effective_cache_size of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.5), pg_size_pretty(sysmem * 0.75); ELSE RAISE NOTICE 'effective_cache_size of % is set appropriately for a system with %', pg_size_pretty(effective_cache_size), pg_size_pretty(sysmem); END IF; IF shared_buffers < (sysmem * 0.2) THEN RAISE WARNING 'shared_buffers of % is set low for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSIF shared_buffers > (sysmem * 0.3) THEN RAISE WARNING 'shared_buffers of % is set high for a system with %. Recomended value between % and %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem), pg_size_pretty(sysmem * 0.2), pg_size_pretty(sysmem * 0.3); ELSE RAISE NOTICE 'shared_buffers of % is set appropriately for a system with %', pg_size_pretty(shared_buffers), pg_size_pretty(sysmem); END IF; shared_buffers = sysmem * 0.3; IF maintenance_work_mem < (sysmem * 0.2) THEN RAISE WARNING 'maintenance_work_mem of % is set low for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSIF maintenance_work_mem > (shared_buffers * 0.3) THEN RAISE WARNING 'maintenance_work_mem of % is set high for shared_buffers of %. Recomended value between % and %', pg_size_pretty(maintenance_work_mem), pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers * 0.2), pg_size_pretty(shared_buffers * 0.3); ELSE RAISE NOTICE 'maintenance_work_mem of % is set appropriately for shared_buffers of %', pg_size_pretty(shared_buffers), pg_size_pretty(shared_buffers); END IF; IF work_mem * max_connections > shared_buffers THEN RAISE WARNING 'work_mem setting of % is set high for % max_connections please reduce work_mem to % or decrease max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSIF work_mem * max_connections < (shared_buffers * 0.75) THEN RAISE WARNING 'work_mem setting of % is set low for % max_connections you may consider raising work_mem to % or increasing max_connections to %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers/max_connections), floor(shared_buffers/work_mem); ELSE RAISE NOTICE 'work_mem setting of % and max_connections of % are adequate for shared_buffers of %', pg_size_pretty(work_mem), max_connections, pg_size_pretty(shared_buffers); END IF; IF random_page_cost / seq_page_cost != 1.1 THEN RAISE WARNING 'random_page_cost (%) /seq_page_cost (%) should be set to 1.1 for SSD. Change random_page_cost to %', random_page_cost, seq_page_cost, 1.1 * seq_page_cost; ELSE RAISE NOTICE 'random_page_cost and seq_page_cost set appropriately for SSD'; END IF; IF temp_buffers < greatest(pg_size_bytes('128MB'),(maintenance_work_mem / 2)) THEN RAISE WARNING 'pgstac makes heavy use of temp tables, consider raising temp_buffers from % to %', pg_size_pretty(temp_buffers), greatest('128MB', pg_size_pretty((shared_buffers / 16))); END IF; END IF; RAISE NOTICE 'VALUES FOR PGSTAC VARIABLES'; RAISE NOTICE 'These can be set either as GUC system variables or by setting in the pgstac_settings table.'; FOR r IN SELECT name, get_setting(name) as setting, CASE WHEN current_setting(concat('pgstac.',name), TRUE) IS NOT NULL THEN concat('pgstac.',name, ' GUC') WHEN value IS NOT NULL THEN 'pgstac_settings table' ELSE 'Not Set' END as loc FROM pgstac_settings LOOP RAISE NOTICE '% is set to % from the %', r.name, r.setting, r.loc; END LOOP; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_cron'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider intalling pg_cron which can be used to automate tasks'; ELSE RAISE NOTICE 'pg_cron % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pgstattuple'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pgstattuple extension which can be used to help maintain tables and indexes.'; ELSE RAISE NOTICE 'pgstattuple % is installed', settingval; END IF; SELECT installed_version INTO settingval from pg_available_extensions WHERE name = 'pg_stat_statements'; IF NOT FOUND OR settingval IS NULL THEN RAISE NOTICE 'Consider installing the pg_stat_statements extension which is very helpful for tracking the types of queries on the system'; ELSE RAISE NOTICE 'pg_stat_statements % is installed', settingval; IF current_setting('pg_stat_statements.track_statements', TRUE) IS DISTINCT FROM 'all' THEN RAISE WARNING 'SET pg_stat_statements.track_statements TO ''all''; --In order to track statements within functions.'; END IF; END IF; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac, public SET CLIENT_MIN_MESSAGES TO NOTICE; ================================================ FILE: src/pgstac/sql/001a_jsonutils.sql ================================================ CREATE OR REPLACE FUNCTION to_int(jsonb) RETURNS int AS $$ SELECT floor(($1->>0)::float)::int; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_float(jsonb) RETURNS float AS $$ SELECT ($1->>0)::float; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_tstz(jsonb) RETURNS timestamptz AS $$ SELECT ($1->>0)::timestamptz; $$ LANGUAGE SQL IMMUTABLE STRICT SET TIME ZONE 'UTC' COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text(jsonb) RETURNS text AS $$ SELECT CASE WHEN jsonb_typeof($1) IN ('array','object') THEN $1::text ELSE $1->>0 END; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION to_text_array(jsonb) RETURNS text[] AS $$ SELECT CASE jsonb_typeof($1) WHEN 'array' THEN ARRAY(SELECT jsonb_array_elements_text($1)) ELSE ARRAY[$1->>0] END ; $$ LANGUAGE SQL IMMUTABLE STRICT COST 5000 PARALLEL SAFE; CREATE OR REPLACE FUNCTION bbox_geom(_bbox jsonb) RETURNS geometry AS $$ SELECT CASE jsonb_array_length(_bbox) WHEN 4 THEN ST_SetSRID(ST_MakeEnvelope( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float, (_bbox->>3)::float ),4326) WHEN 6 THEN ST_SetSRID(ST_3DMakeBox( ST_MakePoint( (_bbox->>0)::float, (_bbox->>1)::float, (_bbox->>2)::float ), ST_MakePoint( (_bbox->>3)::float, (_bbox->>4)::float, (_bbox->>5)::float ) ),4326) ELSE null END; ; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION geom_bbox(_geom geometry) RETURNS jsonb AS $$ SELECT jsonb_build_array( st_xmin(_geom), st_ymin(_geom), st_xmax(_geom), st_ymax(_geom) ); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION flip_jsonb_array(j jsonb) RETURNS jsonb AS $$ SELECT jsonb_agg(value) FROM (SELECT value FROM jsonb_array_elements(j) WITH ORDINALITY ORDER BY ordinality DESC) as t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths(j jsonb) RETURNS SETOF text[] AS $$ SELECT string_to_array(p, '.') as e FROM jsonb_array_elements_text(j) p; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION explode_dotpaths_recurse(IN j jsonb) RETURNS SETOF text[] AS $$ WITH RECURSIVE t AS ( SELECT e FROM explode_dotpaths(j) e UNION ALL SELECT e[1:cardinality(e)-1] FROM t WHERE cardinality(e)>1 ) SELECT e FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_set_nested(j jsonb, path text[], val jsonb) RETURNS jsonb AS $$ DECLARE BEGIN IF cardinality(path) > 1 THEN FOR i IN 1..(cardinality(path)-1) LOOP IF j #> path[:i] IS NULL THEN j := jsonb_set_lax(j, path[:i], '{}', TRUE); END IF; END LOOP; END IF; RETURN jsonb_set_lax(j, path, val, true); END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_include(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE includes jsonb := f-> 'include'; outj jsonb := '{}'::jsonb; path text[]; BEGIN IF includes IS NULL OR jsonb_array_length(includes) = 0 THEN RETURN j; ELSE includes := includes || ( CASE WHEN j ? 'collection' THEN '["id","collection"]' ELSE '["id"]' END)::jsonb; FOR path IN SELECT explode_dotpaths(includes) LOOP outj := jsonb_set_nested(outj, path, j #> path); END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_exclude(j jsonb, f jsonb) RETURNS jsonb AS $$ DECLARE excludes jsonb := f-> 'exclude'; outj jsonb := j; path text[]; BEGIN IF excludes IS NULL OR jsonb_array_length(excludes) = 0 THEN RETURN j; ELSE FOR path IN SELECT explode_dotpaths(excludes) LOOP outj := outj #- path; END LOOP; END IF; RETURN outj; END; $$ LANGUAGE PLPGSQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_fields(j jsonb, f jsonb DEFAULT '{"fields":[]}') RETURNS jsonb AS $$ SELECT jsonb_exclude(jsonb_include(j, f), f); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION merge_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN _a = '"𒍟※"'::jsonb THEN NULL WHEN _a IS NULL OR jsonb_typeof(_a) = 'null' THEN _b WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, merge_jsonb(a.value, b.value) ) ) FROM jsonb_each(coalesce(_a,'{}'::jsonb)) as a FULL JOIN jsonb_each(coalesce(_b,'{}'::jsonb)) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT merge_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION strip_jsonb(_a jsonb, _b jsonb) RETURNS jsonb AS $$ SELECT CASE WHEN (_a IS NULL OR jsonb_typeof(_a) = 'null') AND _b IS NOT NULL AND jsonb_typeof(_b) != 'null' THEN '"𒍟※"'::jsonb WHEN _b IS NULL OR jsonb_typeof(_a) = 'null' THEN _a WHEN _a = _b AND jsonb_typeof(_a) = 'object' THEN '{}'::jsonb WHEN _a = _b THEN NULL WHEN jsonb_typeof(_a) = 'object' AND jsonb_typeof(_b) = 'object' THEN ( SELECT jsonb_strip_nulls( jsonb_object_agg( key, strip_jsonb(a.value, b.value) ) ) FROM jsonb_each(_a) as a FULL JOIN jsonb_each(_b) as b USING (key) ) WHEN jsonb_typeof(_a) = 'array' AND jsonb_typeof(_b) = 'array' AND jsonb_array_length(_a) = jsonb_array_length(_b) THEN ( SELECT jsonb_agg(m) FROM ( SELECT strip_jsonb( jsonb_array_elements(_a), jsonb_array_elements(_b) ) as m ) as l ) ELSE _a END ; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION nullif_jsonbnullempty(j jsonb) RETURNS jsonb AS $$ SELECT nullif(nullif(nullif(j,'null'::jsonb),'{}'::jsonb),'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION jsonb_array_unique(j jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(jsonb_agg(DISTINCT a)) v FROM jsonb_array_elements(j) a; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_concat_ignorenull(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT coalesce(a,'[]'::jsonb) || coalesce(b,'[]'::jsonb); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_least(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(least(nullif_jsonbnullempty(a), nullif_jsonbnullempty(b))); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION jsonb_greatest(a jsonb, b jsonb) RETURNS jsonb AS $$ SELECT nullif_jsonbnullempty(greatest(a, b)); $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION first_notnull_sfunc(anyelement, anyelement) RETURNS anyelement AS $$ SELECT COALESCE($1,$2); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE AGGREGATE first_notnull(anyelement)( SFUNC = first_notnull_sfunc, STYPE = anyelement ); CREATE OR REPLACE AGGREGATE jsonb_array_unique_merge(jsonb) ( STYPE = jsonb, SFUNC = jsonb_concat_ignorenull, FINALFUNC = jsonb_array_unique ); CREATE OR REPLACE AGGREGATE jsonb_min(jsonb) ( STYPE = jsonb, SFUNC = jsonb_least ); CREATE OR REPLACE AGGREGATE jsonb_max(jsonb) ( STYPE = jsonb, SFUNC = jsonb_greatest ); ================================================ FILE: src/pgstac/sql/001s_stacutils.sql ================================================ /* looks for a geometry in a stac item first from geometry and falling back to bbox */ CREATE OR REPLACE FUNCTION stac_geom(value jsonb) RETURNS geometry AS $$ SELECT CASE WHEN value ? 'intersects' THEN ST_GeomFromGeoJSON(value->>'intersects') WHEN value ? 'geometry' THEN ST_GeomFromGeoJSON(value->>'geometry') WHEN value ? 'bbox' THEN pgstac.bbox_geom(value->'bbox') ELSE NULL END as geometry ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION stac_daterange( value jsonb ) RETURNS tstzrange AS $$ DECLARE props jsonb := value; dt timestamptz; edt timestamptz; BEGIN IF props ? 'properties' THEN props := props->'properties'; END IF; IF props ? 'start_datetime' AND props->>'start_datetime' IS NOT NULL AND props ? 'end_datetime' AND props->>'end_datetime' IS NOT NULL THEN dt := props->>'start_datetime'; edt := props->>'end_datetime'; IF dt > edt THEN RAISE EXCEPTION 'start_datetime must be < end_datetime'; END IF; ELSE dt := props->>'datetime'; edt := props->>'datetime'; END IF; IF dt is NULL OR edt IS NULL THEN RAISE NOTICE 'DT: %, EDT: %', dt, edt; RAISE EXCEPTION 'Either datetime (%) or both start_datetime (%) and end_datetime (%) must be set.', props->>'datetime',props->>'start_datetime',props->>'end_datetime'; END IF; RETURN tstzrange(dt, edt, '[]'); END; $$ LANGUAGE PLPGSQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT lower(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE OR REPLACE FUNCTION stac_end_datetime(value jsonb) RETURNS timestamptz AS $$ SELECT upper(stac_daterange(value)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET TIMEZONE='UTC'; CREATE TABLE IF NOT EXISTS stac_extensions( url text PRIMARY KEY, content jsonb ); ================================================ FILE: src/pgstac/sql/002_collections.sql ================================================ CREATE OR REPLACE FUNCTION collection_base_item(content jsonb) RETURNS jsonb AS $$ SELECT jsonb_build_object( 'type', 'Feature', 'stac_version', content->'stac_version', 'assets', content->'item_assets', 'collection', content->'id' ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE IF NOT EXISTS collections ( key bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, id text GENERATED ALWAYS AS (content->>'id') STORED UNIQUE NOT NULL, content JSONB NOT NULL, base_item jsonb GENERATED ALWAYS AS (pgstac.collection_base_item(content)) STORED, geometry geometry GENERATED ALWAYS AS (pgstac.collection_geom(content)) STORED, datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_datetime(content)) STORED, end_datetime timestamptz GENERATED ALWAYS AS (pgstac.collection_enddatetime(content)) STORED, private jsonb, partition_trunc text CHECK (partition_trunc IN ('year', 'month')) ); CREATE OR REPLACE FUNCTION collection_base_item(cid text) RETURNS jsonb AS $$ SELECT pgstac.collection_base_item(content) FROM pgstac.collections WHERE id = cid LIMIT 1; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION create_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_collection(data jsonb) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN UPDATE collections SET content=data WHERE id = data->>'id' RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_collection(data jsonb) RETURNS VOID AS $$ INSERT INTO collections (content) VALUES (data) ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION delete_collection(_id text) RETURNS VOID AS $$ DECLARE out collections%ROWTYPE; BEGIN DELETE FROM collections WHERE id = _id RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION get_collection(id text) RETURNS jsonb AS $$ SELECT content FROM collections WHERE id=$1 ; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION all_collections() RETURNS jsonb AS $$ SELECT coalesce(jsonb_agg(content), '[]'::jsonb) FROM collections; $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_delete_trigger_func() RETURNS TRIGGER AS $$ DECLARE collection_base_partition text := concat('_items_', OLD.key); BEGIN EXECUTE format($q$ DELETE FROM partition_stats WHERE partition IN ( SELECT partition FROM partition_sys_meta WHERE collection=%L ); DROP TABLE IF EXISTS %I CASCADE; $q$, OLD.id, collection_base_partition ); RETURN OLD; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collection_delete_trigger BEFORE DELETE ON collections FOR EACH ROW EXECUTE FUNCTION collection_delete_trigger_func(); ================================================ FILE: src/pgstac/sql/002a_queryables.sql ================================================ CREATE OR REPLACE FUNCTION queryable_signature(n text, c text[]) RETURNS text AS $$ SELECT concat(n, c); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE TABLE queryables ( id bigint GENERATED ALWAYS AS identity PRIMARY KEY, name text NOT NULL, collection_ids text[], -- used to determine what partitions to create indexes on definition jsonb, property_path text, property_wrapper text, property_index_type text ); CREATE INDEX queryables_name_idx ON queryables (name); CREATE INDEX queryables_collection_idx ON queryables USING GIN (collection_ids); CREATE INDEX queryables_property_wrapper_idx ON queryables (property_wrapper); CREATE OR REPLACE FUNCTION pgstac.queryables_constraint_triggerfunc() RETURNS TRIGGER AS $$ DECLARE allcollections text[]; BEGIN RAISE NOTICE 'Making sure that name/collection is unique for queryables %', NEW; IF NEW.collection_ids IS NOT NULL THEN IF EXISTS ( SELECT 1 FROM unnest(NEW.collection_ids) c LEFT JOIN collections ON (collections.id = c) WHERE collections.id IS NULL ) THEN RAISE foreign_key_violation USING MESSAGE = format( 'One or more collections in %s do not exist.', NEW.collection_ids ); RETURN NULL; END IF; END IF; IF TG_OP = 'INSERT' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s: %s', NEW.name, NEW.collection_ids, (SELECT json_agg(row_to_json(q)) FROM queryables q WHERE q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL )) ); RETURN NULL; END IF; END IF; IF TG_OP = 'UPDATE' THEN IF EXISTS ( SELECT 1 FROM queryables q WHERE q.id != NEW.id AND q.name = NEW.name AND ( q.collection_ids && NEW.collection_ids OR q.collection_ids IS NULL OR NEW.collection_ids IS NULL ) ) THEN RAISE unique_violation USING MESSAGE = format( 'There is already a queryable for %s for a collection in %s', NEW.name, NEW.collection_ids ); RETURN NULL; END IF; END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_constraint_insert_trigger BEFORE INSERT ON queryables FOR EACH ROW EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE TRIGGER queryables_constraint_update_trigger BEFORE UPDATE ON queryables FOR EACH ROW WHEN (NEW.name = OLD.name AND NEW.collection_ids IS DISTINCT FROM OLD.collection_ids) EXECUTE PROCEDURE queryables_constraint_triggerfunc(); CREATE OR REPLACE FUNCTION array_to_path(arr text[]) RETURNS text AS $$ SELECT string_agg( quote_literal(v), '->' ) FROM unnest(arr) v; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION queryable( IN dotpath text, OUT path text, OUT expression text, OUT wrapper text, OUT nulled_wrapper text ) AS $$ DECLARE q RECORD; path_elements text[]; BEGIN dotpath := replace(dotpath, 'properties.', ''); IF dotpath = 'start_datetime' THEN dotpath := 'datetime'; END IF; IF dotpath IN ('id', 'geometry', 'datetime', 'end_datetime', 'collection') THEN path := dotpath; expression := dotpath; wrapper := NULL; RETURN; END IF; SELECT * INTO q FROM queryables WHERE name=dotpath OR name = 'properties.' || dotpath OR name = replace(dotpath, 'properties.', '') ; IF q.property_wrapper IS NULL THEN IF q.definition->>'type' = 'number' THEN wrapper := 'to_float'; nulled_wrapper := wrapper; ELSIF q.definition->>'format' = 'date-time' THEN wrapper := 'to_tstz'; nulled_wrapper := wrapper; ELSE nulled_wrapper := NULL; wrapper := 'to_text'; END IF; ELSE wrapper := q.property_wrapper; nulled_wrapper := wrapper; END IF; IF q.property_path IS NOT NULL THEN path := q.property_path; ELSE path_elements := string_to_array(dotpath, '.'); IF path_elements[1] IN ('links', 'assets', 'stac_version', 'stac_extensions') THEN path := format('content->%s', array_to_path(path_elements)); ELSIF path_elements[1] = 'properties' THEN path := format('content->%s', array_to_path(path_elements)); ELSE path := format($F$content->'properties'->%s$F$, array_to_path(path_elements)); END IF; END IF; expression := format('%I(%s)', wrapper, path); RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION unnest_collection(collection_ids text[] DEFAULT NULL) RETURNS SETOF text AS $$ DECLARE BEGIN IF collection_ids IS NULL THEN RETURN QUERY SELECT id FROM collections; END IF; RETURN QUERY SELECT unnest(collection_ids); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION normalize_indexdef(def text) RETURNS text AS $$ DECLARE BEGIN def := btrim(def, ' \n\t'); def := regexp_replace(def, '^CREATE (UNIQUE )?INDEX ([^ ]* )?ON (ONLY )?([^ ]* )?', '', 'i'); RETURN def; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION indexdef(q queryables) RETURNS text AS $$ DECLARE out text; BEGIN IF q.name = 'id' THEN out := 'CREATE UNIQUE INDEX ON %I USING btree (id)'; ELSIF q.name = 'datetime' THEN out := 'CREATE INDEX ON %I USING btree (datetime DESC, end_datetime)'; ELSIF q.name = 'geometry' THEN out := 'CREATE INDEX ON %I USING gist (geometry)'; ELSE out := format($q$CREATE INDEX ON %%I USING %s (%s(((content -> 'properties'::text) -> %L::text)))$q$, lower(COALESCE(q.property_index_type, 'BTREE')), lower(COALESCE(q.property_wrapper, 'to_text')), q.name ); END IF; RETURN btrim(out, ' \n\t'); END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP VIEW IF EXISTS pgstac_indexes; CREATE VIEW pgstac_indexes AS SELECT i.schemaname, i.tablename, i.indexname, regexp_replace(btrim(replace(replace(indexdef, i.indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as idx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty FROM pg_indexes i WHERE i.schemaname='pgstac' and i.tablename ~ '_items_' AND indexdef !~* ' only '; DROP VIEW IF EXISTS pgstac_index_stats; CREATE VIEW pgstac_indexes_stats AS SELECT i.schemaname, i.tablename, i.indexname, indexdef, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime_end_datetime' ELSE NULL END ) AS field, pg_table_size(i.indexname::text) as index_size, pg_size_pretty(pg_table_size(i.indexname::text)) as index_size_pretty, n_distinct, most_common_vals::text::text[], most_common_freqs::text::text[], histogram_bounds::text::text[], correlation FROM pg_indexes i LEFT JOIN pg_stats s ON (s.tablename = i.indexname) WHERE i.schemaname='pgstac' and i.tablename ~ '_items_'; CREATE OR REPLACE FUNCTION queryable_indexes( IN treeroot text DEFAULT 'items', IN changes boolean DEFAULT FALSE, OUT collection text, OUT partition text, OUT field text, OUT indexname text, OUT existing_idx text, OUT queryable_idx text ) RETURNS SETOF RECORD AS $$ WITH p AS ( SELECT relid::text as partition, replace(replace( CASE WHEN parentrelid::regclass::text='items' THEN pg_get_expr(c.relpartbound, c.oid) ELSE pg_get_expr(parent.relpartbound, parent.oid) END, 'FOR VALUES IN (''',''), ''')', '' ) AS collection FROM pg_partition_tree(treeroot) JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) ), i AS ( SELECT partition, indexname, regexp_replace(btrim(replace(replace(indexdef, indexname, ''),'pgstac.',''),' \t\n'), '[ ]+', ' ', 'g') as iidx, COALESCE( (regexp_match(indexdef, '\(([a-zA-Z]+)\)'))[1], (regexp_match(indexdef, '\(content -> ''properties''::text\) -> ''([a-zA-Z0-9\:\_-]+)''::text'))[1], CASE WHEN indexdef ~* '\(datetime desc, end_datetime\)' THEN 'datetime' ELSE NULL END ) AS field FROM pg_indexes JOIN p ON (tablename=partition) ), q AS ( SELECT name AS field, collection, partition, format(indexdef(queryables), partition) as qidx FROM queryables, unnest_collection(queryables.collection_ids) collection JOIN p USING (collection) WHERE property_index_type IS NOT NULL OR name IN ('datetime','geometry','id') ) SELECT collection, partition, field, indexname, iidx as existing_idx, qidx as queryable_idx FROM i FULL JOIN q USING (field, partition) WHERE CASE WHEN changes THEN lower(iidx) IS DISTINCT FROM lower(qidx) ELSE TRUE END; ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION maintain_index( indexname text, queryable_idx text, dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS VOID AS $$ DECLARE BEGIN IF indexname IS NOT NULL THEN IF dropindexes OR queryable_idx IS NOT NULL THEN EXECUTE format('DROP INDEX IF EXISTS %I;', indexname); ELSIF rebuildindexes THEN IF idxconcurrently THEN EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); ELSE EXECUTE format('REINDEX INDEX CONCURRENTLY %I;', indexname); END IF; END IF; END IF; IF queryable_idx IS NOT NULL THEN IF idxconcurrently THEN EXECUTE replace(queryable_idx, 'INDEX', 'INDEX CONCURRENTLY'); ELSE EXECUTE queryable_idx; END IF; END IF; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; set check_function_bodies to off; CREATE OR REPLACE FUNCTION maintain_partition_queries( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE, idxconcurrently boolean DEFAULT FALSE ) RETURNS SETOF text AS $$ DECLARE rec record; q text; BEGIN FOR rec IN ( SELECT * FROM queryable_indexes(part,true) ) LOOP q := format( 'SELECT maintain_index( %L,%L,%L,%L,%L );', rec.indexname, rec.queryable_idx, dropindexes, rebuildindexes, idxconcurrently ); RAISE NOTICE 'Q: %', q; RETURN NEXT q; END LOOP; RETURN; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION maintain_partitions( part text DEFAULT 'items', dropindexes boolean DEFAULT FALSE, rebuildindexes boolean DEFAULT FALSE ) RETURNS VOID AS $$ WITH t AS ( SELECT run_or_queue(q) FROM maintain_partition_queries(part, dropindexes, rebuildindexes) q ) SELECT count(*) FROM t; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION queryables_trigger_func() RETURNS TRIGGER AS $$ DECLARE BEGIN PERFORM maintain_partitions(); RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER queryables_trigger AFTER INSERT OR UPDATE ON queryables FOR EACH STATEMENT EXECUTE PROCEDURE queryables_trigger_func(); CREATE OR REPLACE FUNCTION get_queryables(_collection_ids text[] DEFAULT NULL) RETURNS jsonb AS $$ DECLARE BEGIN -- Build up queryables if the input contains valid collection ids or is empty IF EXISTS ( SELECT 1 FROM collections WHERE _collection_ids IS NULL OR cardinality(_collection_ids) = 0 OR id = ANY(_collection_ids) ) THEN RETURN ( WITH base AS ( SELECT unnest(collection_ids) as collection_id, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE _collection_ids IS NULL OR _collection_ids = '{}'::text[] OR _collection_ids && collection_ids UNION ALL SELECT null, name, coalesce(definition, '{"type":"string"}'::jsonb) as definition FROM queryables WHERE collection_ids IS NULL OR collection_ids = '{}'::text[] ), g AS ( SELECT name, first_notnull(definition) as definition, jsonb_array_unique_merge(definition->'enum') as enum, jsonb_min(definition->'minimum') as minimum, jsonb_min(definition->'maxiumn') as maximum FROM base GROUP BY 1 ) SELECT jsonb_build_object( '$schema', 'http://json-schema.org/draft-07/schema#', '$id', '', 'type', 'object', 'title', 'STAC Queryables.', 'properties', jsonb_object_agg( name, definition || jsonb_strip_nulls(jsonb_build_object( 'enum', enum, 'minimum', minimum, 'maximum', maximum )) ), 'additionalProperties', pgstac.additional_properties() ) FROM g ); ELSE RETURN NULL; END IF; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION get_queryables(_collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT CASE WHEN _collection IS NULL THEN get_queryables(NULL::text[]) ELSE get_queryables(ARRAY[_collection]) END ; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_queryables() RETURNS jsonb AS $$ SELECT get_queryables(NULL::text[]); $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION schema_qualify_refs(url text, j jsonb) returns jsonb as $$ SELECT regexp_replace(j::text, '"\$ref": "#', concat('"$ref": "', url, '#'), 'g')::jsonb; $$ LANGUAGE SQL IMMUTABLE STRICT PARALLEL SAFE; CREATE OR REPLACE VIEW stac_extension_queryables AS SELECT DISTINCT key as name, schema_qualify_refs(e.url, j.value) as definition FROM stac_extensions e, jsonb_each(e.content->'definitions'->'fields'->'properties') j; CREATE OR REPLACE FUNCTION missing_queryables(_collection text, _tablesample float DEFAULT 5, minrows float DEFAULT 10) RETURNS TABLE(collection text, name text, definition jsonb, property_wrapper text) AS $$ DECLARE q text; _partition text; explain_json json; psize float; estrows float; BEGIN SELECT format('_items_%s', key) INTO _partition FROM collections WHERE id=_collection; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM %I;', _partition) INTO explain_json; psize := explain_json->0->'Plan'->'Plan Rows'; estrows := _tablesample * .01 * psize; IF estrows < minrows THEN _tablesample := least(100,greatest(_tablesample, (estrows / psize) / 100)); RAISE NOTICE '%', (psize / estrows) / 100; END IF; RAISE NOTICE 'Using tablesample % to find missing queryables from % % that has ~% rows estrows: %', _tablesample, _collection, _partition, psize, estrows; q := format( $q$ WITH q AS ( SELECT * FROM queryables WHERE collection_ids IS NULL OR %L = ANY(collection_ids) ), t AS ( SELECT content->'properties' AS properties FROM %I TABLESAMPLE SYSTEM(%L) ), p AS ( SELECT DISTINCT ON (key) key, value, s.definition FROM t JOIN LATERAL jsonb_each(properties) ON TRUE LEFT JOIN q ON (q.name=key) LEFT JOIN stac_extension_queryables s ON (s.name=key) WHERE q.definition IS NULL ) SELECT %L, key, COALESCE(definition, jsonb_build_object('type',jsonb_typeof(value))) as definition, CASE WHEN definition->>'type' = 'integer' THEN 'to_int' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'number' THEN 'to_float' WHEN COALESCE(definition->>'type', jsonb_typeof(value)) = 'array' THEN 'to_text_array' ELSE 'to_text' END FROM p; $q$, _collection, _partition, _tablesample, _collection ); RETURN QUERY EXECUTE q; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION missing_queryables(_tablesample float DEFAULT 5) RETURNS TABLE(collection_ids text[], name text, definition jsonb, property_wrapper text) AS $$ SELECT array_agg(collection), name, definition, property_wrapper FROM collections JOIN LATERAL missing_queryables(id, _tablesample) c ON TRUE GROUP BY 2,3,4 ORDER BY 2,1 ; $$ LANGUAGE SQL; ================================================ FILE: src/pgstac/sql/002b_cql.sql ================================================ CREATE OR REPLACE FUNCTION parse_dtrange( _indate jsonb, relative_base timestamptz DEFAULT date_trunc('hour', CURRENT_TIMESTAMP) ) RETURNS tstzrange AS $$ DECLARE timestrs text[]; s timestamptz; e timestamptz; BEGIN timestrs := CASE WHEN _indate ? 'timestamp' THEN ARRAY[_indate->>'timestamp'] WHEN _indate ? 'interval' THEN to_text_array(_indate->'interval') WHEN jsonb_typeof(_indate) = 'array' THEN to_text_array(_indate) ELSE regexp_split_to_array( _indate->>0, '/' ) END; RAISE NOTICE 'TIMESTRS %', timestrs; IF cardinality(timestrs) = 1 THEN IF timestrs[1] ILIKE 'P%' THEN RETURN tstzrange(relative_base - upper(timestrs[1])::interval, relative_base, '[)'); END IF; s := timestrs[1]::timestamptz; RETURN tstzrange(s, s, '[]'); END IF; IF cardinality(timestrs) != 2 THEN RAISE EXCEPTION 'Timestamp cannot have more than 2 values'; END IF; IF timestrs[1] = '..' OR timestrs[1] = '' THEN s := '-infinity'::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] = '..' OR timestrs[2] = '' THEN s := timestrs[1]::timestamptz; e := 'infinity'::timestamptz; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[1] ILIKE 'P%' AND timestrs[2] NOT ILIKE 'P%' THEN e := timestrs[2]::timestamptz; s := e - upper(timestrs[1])::interval; RETURN tstzrange(s,e,'[)'); END IF; IF timestrs[2] ILIKE 'P%' AND timestrs[1] NOT ILIKE 'P%' THEN s := timestrs[1]::timestamptz; e := s + upper(timestrs[2])::interval; RETURN tstzrange(s,e,'[)'); END IF; s := timestrs[1]::timestamptz; e := timestrs[2]::timestamptz; RETURN tstzrange(s,e,'[)'); RETURN NULL; END; $$ LANGUAGE PLPGSQL STABLE STRICT PARALLEL SAFE SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION parse_dtrange( _indate text, relative_base timestamptz DEFAULT CURRENT_TIMESTAMP ) RETURNS tstzrange AS $$ SELECT parse_dtrange(to_jsonb(_indate), relative_base); $$ LANGUAGE SQL STABLE STRICT PARALLEL SAFE; CREATE OR REPLACE FUNCTION temporal_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE ll text := 'datetime'; lh text := 'end_datetime'; rrange tstzrange; rl text; rh text; outq text; BEGIN rrange := parse_dtrange(args->1); RAISE NOTICE 'Constructing temporal query OP: %, ARGS: %, RRANGE: %', op, args, rrange; op := lower(op); rl := format('%L::timestamptz', lower(rrange)); rh := format('%L::timestamptz', upper(rrange)); outq := CASE op WHEN 't_before' THEN 'lh < rl' WHEN 't_after' THEN 'll > rh' WHEN 't_meets' THEN 'lh = rl' WHEN 't_metby' THEN 'll = rh' WHEN 't_overlaps' THEN 'll < rl AND rl < lh < rh' WHEN 't_overlappedby' THEN 'rl < ll < rh AND lh > rh' WHEN 't_starts' THEN 'll = rl AND lh < rh' WHEN 't_startedby' THEN 'll = rl AND lh > rh' WHEN 't_during' THEN 'll > rl AND lh < rh' WHEN 't_contains' THEN 'll < rl AND lh > rh' WHEN 't_finishes' THEN 'll > rl AND lh = rh' WHEN 't_finishedby' THEN 'll < rl AND lh = rh' WHEN 't_equals' THEN 'll = rl AND lh = rh' WHEN 't_disjoint' THEN 'NOT (ll <= rh AND lh >= rl)' WHEN 't_intersects' THEN 'll <= rh AND lh >= rl' WHEN 'anyinteracts' THEN 'll <= rh AND lh >= rl' END; outq := regexp_replace(outq, '\mll\M', ll); outq := regexp_replace(outq, '\mlh\M', lh); outq := regexp_replace(outq, '\mrl\M', rl); outq := regexp_replace(outq, '\mrh\M', rh); outq := format('(%s)', outq); RETURN outq; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION spatial_op_query(op text, args jsonb) RETURNS text AS $$ DECLARE geom text; j jsonb := args->1; BEGIN op := lower(op); RAISE NOTICE 'Constructing spatial query OP: %, ARGS: %', op, args; IF op NOT IN ('s_equals','s_disjoint','s_touches','s_within','s_overlaps','s_crosses','s_intersects','intersects','s_contains') THEN RAISE EXCEPTION 'Spatial Operator % Not Supported', op; END IF; op := regexp_replace(op, '^s_', 'st_'); IF op = 'intersects' THEN op := 'st_intersects'; END IF; -- Convert geometry to WKB string IF j ? 'type' AND j ? 'coordinates' THEN geom := st_geomfromgeojson(j)::text; ELSIF jsonb_typeof(j) = 'array' THEN geom := bbox_geom(j)::text; END IF; RETURN format('%s(geometry, %L::geometry)', op, geom); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION query_to_cql2(q jsonb) RETURNS jsonb AS $$ -- Translates anything passed in through the deprecated "query" into equivalent CQL2 WITH t AS ( SELECT key as property, value as ops FROM jsonb_each(q) ), t2 AS ( SELECT property, (jsonb_each(ops)).* FROM t WHERE jsonb_typeof(ops) = 'object' UNION ALL SELECT property, 'eq', ops FROM t WHERE jsonb_typeof(ops) != 'object' ) SELECT jsonb_strip_nulls(jsonb_build_object( 'op', 'and', 'args', jsonb_agg( jsonb_build_object( 'op', key, 'args', jsonb_build_array( jsonb_build_object('property',property), value ) ) ) ) ) as qcql FROM t2 ; $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION cql1_to_cql2(j jsonb) RETURNS jsonb AS $$ DECLARE args jsonb; ret jsonb; BEGIN RAISE NOTICE 'CQL1_TO_CQL2: %', j; IF j ? 'filter' THEN RETURN cql1_to_cql2(j->'filter'); END IF; IF j ? 'property' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'array' THEN SELECT jsonb_agg(cql1_to_cql2(el)) INTO args FROM jsonb_array_elements(j) el; RETURN args; END IF; IF jsonb_typeof(j) = 'number' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'string' THEN RETURN j; END IF; IF jsonb_typeof(j) = 'object' THEN SELECT jsonb_build_object( 'op', key, 'args', cql1_to_cql2(value) ) INTO ret FROM jsonb_each(j) WHERE j IS NOT NULL; RETURN ret; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL IMMUTABLE STRICT; CREATE TABLE cql2_ops ( op text PRIMARY KEY, template text, types text[] ); INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; CREATE OR REPLACE FUNCTION cql2_query(j jsonb, wrapper text DEFAULT NULL) RETURNS text AS $$ #variable_conflict use_variable DECLARE args jsonb := j->'args'; arg jsonb; op text := lower(j->>'op'); cql2op RECORD; literal text; _wrapper text; leftarg text; rightarg text; prop text; extra_props bool := pgstac.additional_properties(); BEGIN IF j IS NULL OR (op IS NOT NULL AND args IS NULL) THEN RETURN NULL; END IF; RAISE NOTICE 'CQL2_QUERY: %', j; -- check if all properties are represented in the queryables IF NOT extra_props THEN FOR prop IN SELECT DISTINCT p->>0 FROM jsonb_path_query(j, 'strict $.**.property') p WHERE p->>0 NOT IN ('id', 'datetime', 'geometry', 'end_datetime', 'collection') LOOP IF (queryable(prop)).nulled_wrapper IS NULL THEN RAISE EXCEPTION 'Term % is not found in queryables.', prop; END IF; END LOOP; END IF; IF j ? 'filter' THEN RETURN cql2_query(j->'filter'); END IF; IF j ? 'upper' THEN RETURN cql2_query(jsonb_build_object('op', 'upper', 'args', j->'upper')); END IF; IF j ? 'lower' THEN RETURN cql2_query(jsonb_build_object('op', 'lower', 'args', j->'lower')); END IF; -- Temporal Query IF op ilike 't_%' or op = 'anyinteracts' THEN RETURN temporal_op_query(op, args); END IF; -- If property is a timestamp convert it to text to use with -- general operators IF j ? 'timestamp' THEN RETURN format('%L::timestamptz', to_tstz(j->'timestamp')); END IF; IF j ? 'interval' THEN RAISE EXCEPTION 'Please use temporal operators when using intervals.'; RETURN NONE; END IF; -- Spatial Query IF op ilike 's_%' or op = 'intersects' THEN RETURN spatial_op_query(op, args); END IF; IF op IN ('a_equals','a_contains','a_contained_by','a_overlaps') THEN IF args->0 ? 'property' THEN leftarg := format('to_text_array(%s)', (queryable(args->0->>'property')).path); END IF; IF args->1 ? 'property' THEN rightarg := format('to_text_array(%s)', (queryable(args->1->>'property')).path); END IF; RETURN FORMAT( '%s %s %s', COALESCE(leftarg, quote_literal(to_text_array(args->0))), CASE op WHEN 'a_equals' THEN '=' WHEN 'a_contains' THEN '@>' WHEN 'a_contained_by' THEN '<@' WHEN 'a_overlaps' THEN '&&' END, COALESCE(rightarg, quote_literal(to_text_array(args->1))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN : % % %', args, jsonb_build_array(args->0), args->1; args := jsonb_build_array(args->0) || (args->1); RAISE NOTICE 'IN2 : %', args; END IF; IF op = 'between' THEN args = jsonb_build_array( args->0, args->1, args->2 ); END IF; -- Make sure that args is an array and run cql2_query on -- each element of the array RAISE NOTICE 'ARGS PRE: %', args; IF j ? 'args' THEN IF jsonb_typeof(args) != 'array' THEN args := jsonb_build_array(args); END IF; IF jsonb_path_exists(args, '$[*] ? (@.property == "id" || @.property == "datetime" || @.property == "end_datetime" || @.property == "collection")') THEN wrapper := NULL; ELSE -- if any of the arguments are a property, try to get the property_wrapper FOR arg IN SELECT jsonb_path_query(args, '$[*] ? (@.property != null)') LOOP RAISE NOTICE 'Arg: %', arg; wrapper := (queryable(arg->>'property')).nulled_wrapper; RAISE NOTICE 'Property: %, Wrapper: %', arg, wrapper; IF wrapper IS NOT NULL THEN EXIT; END IF; END LOOP; -- if the property was not in queryables, see if any args were numbers IF wrapper IS NULL AND jsonb_path_exists(args, '$[*] ? (@.type()=="number")') THEN wrapper := 'to_float'; END IF; wrapper := coalesce(wrapper, 'to_text'); END IF; SELECT jsonb_agg(cql2_query(a, wrapper)) INTO args FROM jsonb_array_elements(args) a; END IF; RAISE NOTICE 'ARGS: %', args; IF op IN ('and', 'or') THEN RETURN format( '(%s)', array_to_string(to_text_array(args), format(' %s ', upper(op))) ); END IF; IF op = 'in' THEN RAISE NOTICE 'IN -- % %', args->0, to_text(args->0); RETURN format( '%s IN (%s)', to_text(args->0), array_to_string((to_text_array(args))[2:], ',') ); END IF; -- Look up template from cql2_ops IF j ? 'op' THEN SELECT * INTO cql2op FROM cql2_ops WHERE cql2_ops.op ilike op; IF FOUND THEN -- If specific index set in queryables for a property cast other arguments to that type RETURN format( cql2op.template, VARIADIC (to_text_array(args)) ); ELSE RAISE EXCEPTION 'Operator % Not Supported.', op; END IF; END IF; IF wrapper IS NOT NULL THEN RAISE NOTICE 'Wrapping % with %', j, wrapper; IF j ? 'property' THEN RETURN format('%I(%s)', wrapper, (queryable(j->>'property')).path); ELSE RETURN format('%I(%L)', wrapper, j); END IF; ELSIF j ? 'property' THEN RETURN quote_ident(j->>'property'); END IF; RETURN quote_literal(to_text(j)); END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION paging_dtrange( j jsonb ) RETURNS tstzrange AS $$ DECLARE op text; filter jsonb := j->'filter'; dtrange tstzrange := tstzrange('-infinity'::timestamptz,'infinity'::timestamptz); sdate timestamptz := '-infinity'::timestamptz; edate timestamptz := 'infinity'::timestamptz; jpitem jsonb; BEGIN IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "datetime")'::jsonpath) j LOOP op := lower(jpitem->>'op'); dtrange := parse_dtrange(jpitem->'args'->1); IF op IN ('<=', 'lt', 'lte', '<', 'le', 't_before') THEN sdate := greatest(sdate,'-infinity'); edate := least(edate, upper(dtrange)); ELSIF op IN ('>=', '>', 'gt', 'gte', 'ge', 't_after') THEN edate := least(edate, 'infinity'); sdate := greatest(sdate, lower(dtrange)); ELSIF op IN ('=', 'eq') THEN edate := least(edate, upper(dtrange)); sdate := greatest(sdate, lower(dtrange)); END IF; RAISE NOTICE '2 OP: %, ARGS: %, DTRANGE: %, SDATE: %, EDATE: %', op, jpitem->'args'->1, dtrange, sdate, edate; END LOOP; END IF; IF sdate > edate THEN RETURN 'empty'::tstzrange; END IF; RETURN tstzrange(sdate,edate, '[]'); END; $$ LANGUAGE PLPGSQL STABLE STRICT SET TIME ZONE 'UTC'; CREATE OR REPLACE FUNCTION paging_collections( IN j jsonb ) RETURNS text[] AS $$ DECLARE filter jsonb := j->'filter'; jpitem jsonb; op text; args jsonb; arg jsonb; collections text[]; BEGIN IF j ? 'collections' THEN collections := to_text_array(j->'collections'); END IF; IF NOT (filter @? '$.**.op ? (@ == "or" || @ == "not")') THEN FOR jpitem IN SELECT j FROM jsonb_path_query(filter,'strict $.** ? (@.args[*].property == "collection")'::jsonpath) j LOOP RAISE NOTICE 'JPITEM: %', jpitem; op := jpitem->>'op'; args := jpitem->'args'; IF op IN ('=', 'eq', 'in') THEN FOR arg IN SELECT a FROM jsonb_array_elements(args) a LOOP IF jsonb_typeof(arg) IN ('string', 'array') THEN RAISE NOTICE 'arg: %, collections: %', arg, collections; IF collections IS NULL OR collections = '{}'::text[] THEN collections := to_text_array(arg); ELSE collections := array_intersection(collections, to_text_array(arg)); END IF; END IF; END LOOP; END IF; END LOOP; END IF; IF collections = '{}'::text[] THEN RETURN NULL; END IF; RETURN collections; END; $$ LANGUAGE PLPGSQL STABLE STRICT; ================================================ FILE: src/pgstac/sql/003a_items.sql ================================================ CREATE TABLE items ( id text NOT NULL, geometry geometry NOT NULL, collection text NOT NULL, datetime timestamptz NOT NULL, end_datetime timestamptz NOT NULL, content JSONB NOT NULL, private jsonb ) PARTITION BY LIST (collection) ; CREATE INDEX "datetime_idx" ON items USING BTREE (datetime DESC, end_datetime ASC); CREATE INDEX "geometry_idx" ON items USING GIST (geometry); CREATE STATISTICS datetime_stats (dependencies) on datetime, end_datetime from items; ALTER TABLE items ADD CONSTRAINT items_collections_fk FOREIGN KEY (collection) REFERENCES collections(id) ON DELETE CASCADE DEFERRABLE; CREATE OR REPLACE FUNCTION partition_after_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p text; t timestamptz := clock_timestamp(); BEGIN RAISE NOTICE 'Updating partition stats %', t; FOR p IN SELECT DISTINCT partition FROM newdata n JOIN partition_sys_meta p ON (n.collection=p.collection AND n.datetime <@ p.partition_dtrange) LOOP PERFORM run_or_queue(format('SELECT update_partition_stats(%L, %L);', p, true)); END LOOP; IF TG_OP IN ('DELETE','UPDATE') THEN DELETE FROM format_item_cache c USING newdata n WHERE c.collection = n.collection AND c.id = n.id; END IF; RAISE NOTICE 't: % %', t, clock_timestamp() - t; RETURN NULL; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE TRIGGER items_after_insert_trigger AFTER INSERT ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_update_trigger AFTER DELETE ON items REFERENCING OLD TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE TRIGGER items_after_delete_trigger AFTER UPDATE ON items REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE FUNCTION partition_after_triggerfunc(); CREATE OR REPLACE FUNCTION content_slim(_item jsonb) RETURNS jsonb AS $$ SELECT strip_jsonb(_item - '{id,geometry,collection,type}'::text[], collection_base_item(_item->>'collection')) - '{id,geometry,collection,type}'::text[]; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_dehydrate(content jsonb) RETURNS items AS $$ SELECT content->>'id' as id, stac_geom(content) as geometry, content->>'collection' as collection, stac_datetime(content) as datetime, stac_end_datetime(content) as end_datetime, content_slim(content) as content, null::jsonb as private ; $$ LANGUAGE SQL STABLE; CREATE OR REPLACE FUNCTION include_field(f text, fields jsonb DEFAULT '{}'::jsonb) RETURNS boolean AS $$ DECLARE includes jsonb := fields->'include'; excludes jsonb := fields->'exclude'; BEGIN IF f IS NULL THEN RETURN NULL; END IF; IF jsonb_typeof(excludes) = 'array' AND jsonb_array_length(excludes)>0 AND excludes ? f THEN RETURN FALSE; END IF; IF ( jsonb_typeof(includes) = 'array' AND jsonb_array_length(includes) > 0 AND includes ? f ) OR ( includes IS NULL OR jsonb_typeof(includes) = 'null' OR jsonb_array_length(includes) = 0 ) THEN RETURN TRUE; END IF; RETURN FALSE; END; $$ LANGUAGE PLPGSQL IMMUTABLE; DROP FUNCTION IF EXISTS content_hydrate(jsonb, jsonb, jsonb); CREATE OR REPLACE FUNCTION content_hydrate( _item jsonb, _base_item jsonb, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ SELECT merge_jsonb( jsonb_fields(_item, fields), jsonb_fields(_base_item, fields) ); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, _collection collections, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; content jsonb; base_item jsonb := _collection.base_item; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := content_hydrate( jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content, _collection.base_item, fields ); RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_nonhydrated( _item items, fields jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE geom jsonb; bbox jsonb; output jsonb; BEGIN IF include_field('geometry', fields) THEN geom := ST_ASGeoJson(_item.geometry, 20)::jsonb; END IF; output := jsonb_build_object( 'id', _item.id, 'geometry', geom, 'collection', _item.collection, 'type', 'Feature' ) || _item.content; RETURN output; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION content_hydrate(_item items, fields jsonb DEFAULT '{}'::jsonb) RETURNS jsonb AS $$ SELECT content_hydrate( _item, (SELECT c FROM collections c WHERE id=_item.collection LIMIT 1), fields ); $$ LANGUAGE SQL STABLE; CREATE UNLOGGED TABLE items_staging ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_ignore ( content JSONB NOT NULL ); CREATE UNLOGGED TABLE items_staging_upsert ( content JSONB NOT NULL ); CREATE OR REPLACE FUNCTION items_staging_triggerfunc() RETURNS TRIGGER AS $$ DECLARE p record; _partitions text[]; part text; ts timestamptz := clock_timestamp(); nrows int; BEGIN RAISE NOTICE 'Creating Partitions. %', clock_timestamp() - ts; FOR part IN WITH t AS ( SELECT n.content->>'collection' as collection, stac_daterange(n.content->'properties') as dtr, partition_trunc FROM newdata n JOIN collections ON (n.content->>'collection'=collections.id) ), p AS ( SELECT collection, COALESCE(date_trunc(partition_trunc::text, lower(dtr)),'-infinity') as d, tstzrange(min(lower(dtr)),max(lower(dtr)),'[]') as dtrange, tstzrange(min(upper(dtr)),max(upper(dtr)),'[]') as edtrange FROM t GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p LOOP RAISE NOTICE 'Partition %', part; END LOOP; RAISE NOTICE 'Creating temp table with data to be added. %', clock_timestamp() - ts; DROP TABLE IF EXISTS tmpdata; CREATE TEMP TABLE tmpdata ON COMMIT DROP AS SELECT (content_dehydrate(content)).* FROM newdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Added % rows to tmpdata. %', nrows, clock_timestamp() - ts; RAISE NOTICE 'Doing the insert. %', clock_timestamp() - ts; IF TG_TABLE_NAME = 'items_staging' THEN INSERT INTO items SELECT * FROM tmpdata; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_ignore' THEN INSERT INTO items SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; ELSIF TG_TABLE_NAME = 'items_staging_upsert' THEN DELETE FROM items i USING tmpdata s WHERE i.id = s.id AND i.collection = s.collection AND i IS DISTINCT FROM s ; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Deleted % rows from items. %', nrows, clock_timestamp() - ts; INSERT INTO items AS t SELECT * FROM tmpdata ON CONFLICT DO NOTHING; GET DIAGNOSTICS nrows = ROW_COUNT; RAISE NOTICE 'Inserted % rows to items. %', nrows, clock_timestamp() - ts; END IF; RAISE NOTICE 'Deleting data from staging table. %', clock_timestamp() - ts; DELETE FROM items_staging; RAISE NOTICE 'Done. %', clock_timestamp() - ts; RETURN NULL; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER items_staging_insert_trigger AFTER INSERT ON items_staging REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_ignore_trigger AFTER INSERT ON items_staging_ignore REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE TRIGGER items_staging_insert_upsert_trigger AFTER INSERT ON items_staging_upsert REFERENCING NEW TABLE AS newdata FOR EACH STATEMENT EXECUTE PROCEDURE items_staging_triggerfunc(); CREATE OR REPLACE FUNCTION item_by_id(_id text, _collection text DEFAULT NULL) RETURNS items AS $$ DECLARE i items%ROWTYPE; BEGIN SELECT * INTO i FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection) LIMIT 1; RETURN i; END; $$ LANGUAGE PLPGSQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION get_item(_id text, _collection text DEFAULT NULL) RETURNS jsonb AS $$ SELECT content_hydrate(items) FROM items WHERE id=_id AND (_collection IS NULL OR collection=_collection); $$ LANGUAGE SQL STABLE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION delete_item(_id text, _collection text DEFAULT NULL) RETURNS VOID AS $$ DECLARE out items%ROWTYPE; BEGIN DELETE FROM items WHERE id = _id AND (_collection IS NULL OR collection=_collection) RETURNING * INTO STRICT out; END; $$ LANGUAGE PLPGSQL; --/* CREATE OR REPLACE FUNCTION create_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION update_item(content jsonb) RETURNS VOID AS $$ DECLARE old items %ROWTYPE; out items%ROWTYPE; BEGIN PERFORM delete_item(content->>'id', content->>'collection'); PERFORM create_item(content); END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_item(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) VALUES (data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION create_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION upsert_items(data jsonb) RETURNS VOID AS $$ INSERT INTO items_staging_upsert (content) SELECT * FROM jsonb_array_elements(data); $$ LANGUAGE SQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION collection_bbox(id text) RETURNS jsonb AS $$ SELECT (replace(replace(replace(st_extent(geometry)::text,'BOX(','[['),')',']]'),' ',','))::jsonb FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION collection_temporal_extent(id text) RETURNS jsonb AS $$ SELECT to_jsonb(array[array[min(datetime), max(datetime)]]) FROM items WHERE collection=$1; ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION update_collection_extents() RETURNS VOID AS $$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(id, FALSE), true, 'use_json_null' ) ; $$ LANGUAGE SQL; ================================================ FILE: src/pgstac/sql/003b_partitions.sql ================================================ CREATE TABLE partition_stats ( partition text PRIMARY KEY, dtrange tstzrange, edtrange tstzrange, spatial geometry, last_updated timestamptz, keys text[] ) WITH (FILLFACTOR=90); CREATE INDEX partitions_range_idx ON partition_stats USING GIST(dtrange); CREATE OR REPLACE FUNCTION constraint_tstzrange(expr text) RETURNS tstzrange AS $$ WITH t AS ( SELECT regexp_matches( expr, E'\\(''\([0-9 :+-]*\)''\\).*\\(''\([0-9 :+-]*\)''\\)' ) AS m ) SELECT tstzrange(m[1]::timestamptz, m[2]::timestamptz) FROM t ; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE STRICT; CREATE OR REPLACE FUNCTION get_tstz_constraint(reloid oid, colname text) RETURNS tstzrange AS $$ DECLARE expr text := NULL; m text[]; ts_lower timestamptz := NULL; ts_upper timestamptz := NULL; lower_inclusive text := '['; upper_inclusive text := ']'; ts timestamptz; BEGIN SELECT INTO expr string_agg(def, ' AND ') FROM pg_constraint JOIN LATERAL pg_get_constraintdef(oid) AS def ON TRUE WHERE conrelid = reloid AND contype = 'c' AND def LIKE '%' || colname || '%' ; IF expr IS NULL THEN RETURN NULL; END IF; RAISE DEBUG 'Constraint expression for % on %: %', colname, reloid::regclass, expr; -- collect all constraints for the specified column FOR m IN SELECT regexp_matches(expr, '[ (]' || colname || $expr$\s*([<>=]{1,2})\s*'([0-9 :.+\-]+)'$expr$, 'g') LOOP ts := m[2]::timestamptz; IF m[1] IN ('>', '>=') THEN IF ts_lower IS NULL OR ts > ts_lower OR (ts = ts_lower AND m[1] = '>') THEN ts_lower := ts; lower_inclusive := CASE WHEN m[1] = '>' THEN '(' ELSE '[' END; END IF; ELSIF m[1] IN ('<', '<=') THEN IF ts_upper IS NULL OR ts < ts_upper OR (ts = ts_upper AND m[1] = '<') THEN ts_upper := ts; upper_inclusive := CASE WHEN m[1] = '<' THEN ')' ELSE ']' END; END IF; END IF; END LOOP; RAISE DEBUG 'Constraint % for %: % %', colname, reloid::regclass, ts_lower, ts_upper; RETURN tstzrange(ts_lower, ts_upper, lower_inclusive || upper_inclusive); END; $$ LANGUAGE plpgsql STRICT STABLE; CREATE OR REPLACE FUNCTION get_partition_name(relid regclass) RETURNS text AS $$ SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))]; $$ LANGUAGE SQL STABLE STRICT; CREATE OR REPLACE VIEW partition_sys_meta AS SELECT partition, replace( replace( CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, 'FOR VALUES IN (''', '' ), ''')', '' ) AS collection, level, c.reltuples, c.relhastriggers, partition_dtrange, COALESCE( get_tstz_constraint(c.oid, 'datetime'), partition_dtrange, inf_range ) as constraint_dtrange, COALESCE( get_tstz_constraint(c.oid, 'end_datetime'), inf_range ) as constraint_edtrange FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE WHERE isleaf ; CREATE OR REPLACE VIEW partitions_view AS SELECT (parse_ident(relid::text))[cardinality(parse_ident(relid::text))] as partition, replace( replace( CASE WHEN level = 1 THEN partition_expr ELSE parent_partition_expr END, 'FOR VALUES IN (''', '' ), ''')', '' ) AS collection, level, c.reltuples, c.relhastriggers, partition_dtrange, COALESCE( get_tstz_constraint(c.oid, 'datetime'), partition_dtrange, inf_range ) as constraint_dtrange, COALESCE( get_tstz_constraint(c.oid, 'end_datetime'), inf_range ) as constraint_edtrange, dtrange, edtrange, spatial, last_updated FROM pg_partition_tree('items') JOIN pg_class c ON (relid::regclass = c.oid) JOIN pg_class parent ON (parentrelid::regclass = parent.oid AND isleaf) LEFT JOIN pg_constraint edt ON (conrelid=c.oid AND contype='c') JOIN LATERAL get_partition_name(relid) AS partition ON TRUE JOIN LATERAL pg_get_expr(c.relpartbound, c.oid) as partition_expr ON TRUE JOIN LATERAL pg_get_expr(parent.relpartbound, parent.oid) as parent_partition_expr ON TRUE JOIN LATERAL tstzrange('-infinity', 'infinity','[]') as inf_range ON TRUE JOIN LATERAL COALESCE(constraint_tstzrange(pg_get_expr(c.relpartbound, c.oid)), inf_range) as partition_dtrange ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'datetime') as datetime_constraint ON TRUE JOIN LATERAL get_tstz_constraint(c.oid, 'end_datetime') as end_datetime_constraint ON TRUE LEFT JOIN pgstac.partition_stats USING (partition) WHERE isleaf ; CREATE MATERIALIZED VIEW partitions AS SELECT * FROM partitions_view; CREATE UNIQUE INDEX ON partitions (partition); CREATE MATERIALIZED VIEW partition_steps AS SELECT partition as name, date_trunc('month',lower(partition_dtrange)) as sdate, date_trunc('month', upper(partition_dtrange)) + '1 month'::interval as edate FROM partitions_view WHERE partition_dtrange IS NOT NULL AND partition_dtrange != 'empty'::tstzrange ORDER BY dtrange ASC ; CREATE OR REPLACE FUNCTION update_partition_stats_q(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE BEGIN PERFORM run_or_queue( format('SELECT update_partition_stats(%L, %L);', _partition, istrigger) ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION update_partition_stats(_partition text, istrigger boolean default false) RETURNS VOID AS $$ DECLARE dtrange tstzrange; edtrange tstzrange; cdtrange tstzrange; cedtrange tstzrange; extent geometry; collection text; BEGIN RAISE NOTICE 'Updating stats for %.', _partition; EXECUTE format( $q$ SELECT tstzrange(min(datetime), max(datetime),'[]'), tstzrange(min(end_datetime), max(end_datetime), '[]') FROM %I $q$, _partition ) INTO dtrange, edtrange; EXECUTE format('ANALYZE %I;', _partition); extent := st_estimatedextent('pgstac', _partition, 'geometry'); RAISE DEBUG 'Estimated Extent: %', extent; INSERT INTO partition_stats (partition, dtrange, edtrange, spatial, last_updated) SELECT _partition, dtrange, edtrange, extent, now() ON CONFLICT (partition) DO UPDATE SET dtrange=EXCLUDED.dtrange, edtrange=EXCLUDED.edtrange, spatial=EXCLUDED.spatial, last_updated=EXCLUDED.last_updated ; SELECT constraint_dtrange, constraint_edtrange, pv.collection INTO cdtrange, cedtrange, collection FROM partitions_view pv WHERE partition = _partition; RAISE NOTICE 'Checking if we need to modify constraints...'; RAISE NOTICE 'cdtrange: % dtrange: % cedtrange: % edtrange: %',cdtrange, dtrange, cedtrange, edtrange; IF (cdtrange IS DISTINCT FROM dtrange OR edtrange IS DISTINCT FROM cedtrange) AND NOT istrigger THEN RAISE NOTICE 'Modifying Constraints'; RAISE NOTICE 'Existing % %', cdtrange, cedtrange; RAISE NOTICE 'New % %', dtrange, edtrange; PERFORM drop_table_constraints(_partition); PERFORM create_table_constraints(_partition, dtrange, edtrange); END IF; REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RAISE NOTICE 'Checking if we need to update collection extents.'; IF get_setting_bool('update_collection_extent') THEN RAISE NOTICE 'updating collection extent for %', collection; PERFORM run_or_queue(format($q$ UPDATE collections SET content = jsonb_set_lax( content, '{extent}'::text[], collection_extent(%L, FALSE), true, 'use_json_null' ) WHERE id=%L ; $q$, collection, collection)); ELSE RAISE NOTICE 'Not updating collection extent for %', collection; END IF; END; $$ LANGUAGE PLPGSQL STRICT SECURITY DEFINER; CREATE OR REPLACE FUNCTION partition_name( IN collection text, IN dt timestamptz, OUT partition_name text, OUT partition_range tstzrange) AS $$ DECLARE c RECORD; parent_name text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; parent_name := format('_items_%s', c.key); IF c.partition_trunc = 'year' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYY')); ELSIF c.partition_trunc = 'month' THEN partition_name := format('%s_%s', parent_name, to_char(dt,'YYYYMM')); ELSE partition_name := parent_name; partition_range := tstzrange('-infinity'::timestamptz, 'infinity'::timestamptz, '[]'); END IF; IF partition_range IS NULL THEN partition_range := tstzrange( date_trunc(c.partition_trunc::text, dt), date_trunc(c.partition_trunc::text, dt) + concat('1 ', c.partition_trunc)::interval ); END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION drop_table_constraints(t text) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; FOR q IN SELECT FORMAT( $q$ ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; $q$, t, conname ) FROM pg_constraint WHERE conrelid=t::regclass::oid AND contype='c' LOOP EXECUTE q; END LOOP; RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION create_table_constraints(t text, _dtrange tstzrange, _edtrange tstzrange) RETURNS text AS $$ DECLARE q text; BEGIN IF NOT EXISTS (SELECT 1 FROM partitions_view WHERE partition=t) THEN RETURN NULL; END IF; RAISE NOTICE 'Creating Table Constraints for % % %', t, _dtrange, _edtrange; IF _dtrange = 'empty' AND _edtrange = 'empty' THEN q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK (((datetime IS NULL) AND (end_datetime IS NULL))) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), t, format('%s_dt', t), t ); ELSE q :=format( $q$ DO $block$ BEGIN ALTER TABLE %I DROP CONSTRAINT IF EXISTS %I; ALTER TABLE %I ADD CONSTRAINT %I CHECK ( (datetime >= %L) AND (datetime <= %L) AND (end_datetime >= %L) AND (end_datetime <= %L) ) NOT VALID ; ALTER TABLE %I VALIDATE CONSTRAINT %I ; EXCEPTION WHEN others THEN RAISE WARNING '%%, Issue Altering Constraints. Please run update_partition_stats(%I)', SQLERRM USING ERRCODE = SQLSTATE; END; $block$; $q$, t, format('%s_dt', t), t, format('%s_dt', t), lower(_dtrange), upper(_dtrange), lower(_edtrange), upper(_edtrange), t, format('%s_dt', t), t ); END IF; PERFORM run_or_queue(q); RETURN t; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION check_partition( _collection text, _dtrange tstzrange, _edtrange tstzrange ) RETURNS text AS $$ DECLARE c RECORD; pm RECORD; _partition_name text; _partition_dtrange tstzrange; _constraint_dtrange tstzrange; _constraint_edtrange tstzrange; q text; deferrable_q text; err_context text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF c.partition_trunc IS NOT NULL THEN _partition_dtrange := tstzrange( date_trunc(c.partition_trunc, lower(_dtrange)), date_trunc(c.partition_trunc, lower(_dtrange)) + (concat('1 ', c.partition_trunc))::interval, '[)' ); ELSE _partition_dtrange := '[-infinity, infinity]'::tstzrange; END IF; IF NOT _partition_dtrange @> _dtrange THEN RAISE EXCEPTION 'dtrange % is greater than the partition size % for collection %', _dtrange, c.partition_trunc, _collection; END IF; IF c.partition_trunc = 'year' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYY')); ELSIF c.partition_trunc = 'month' THEN _partition_name := format('_items_%s_%s', c.key, to_char(lower(_partition_dtrange),'YYYYMM')); ELSE _partition_name := format('_items_%s', c.key); END IF; SELECT * INTO pm FROM partition_sys_meta WHERE collection=_collection AND partition_dtrange @> _dtrange; IF FOUND THEN RAISE NOTICE '% % %', _edtrange, _dtrange, pm; _constraint_edtrange := tstzrange( least( lower(_edtrange), nullif(lower(pm.constraint_edtrange), '-infinity') ), greatest( upper(_edtrange), nullif(upper(pm.constraint_edtrange), 'infinity') ), '[]' ); _constraint_dtrange := tstzrange( least( lower(_dtrange), nullif(lower(pm.constraint_dtrange), '-infinity') ), greatest( upper(_dtrange), nullif(upper(pm.constraint_dtrange), 'infinity') ), '[]' ); IF pm.constraint_edtrange @> _edtrange AND pm.constraint_dtrange @> _dtrange THEN RETURN pm.partition; ELSE PERFORM drop_table_constraints(_partition_name); END IF; ELSE _constraint_edtrange := _edtrange; _constraint_dtrange := _dtrange; END IF; RAISE NOTICE 'EXISTING CONSTRAINTS % %, NEW % %', pm.constraint_dtrange, pm.constraint_edtrange, _constraint_dtrange, _constraint_edtrange; RAISE NOTICE 'Creating partition % %', _partition_name, _partition_dtrange; IF c.partition_trunc IS NULL THEN q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I to pgstac_ingest; $q$, _partition_name, _collection, concat(_partition_name,'_pk'), _partition_name, _partition_name ); ELSE q := format( $q$ CREATE TABLE IF NOT EXISTS %I partition OF items FOR VALUES IN (%L) PARTITION BY RANGE (datetime); CREATE TABLE IF NOT EXISTS %I partition OF %I FOR VALUES FROM (%L) TO (%L); CREATE UNIQUE INDEX IF NOT EXISTS %I ON %I (id); GRANT ALL ON %I TO pgstac_ingest; $q$, format('_items_%s', c.key), _collection, _partition_name, format('_items_%s', c.key), lower(_partition_dtrange), upper(_partition_dtrange), format('%s_pk', _partition_name), _partition_name, _partition_name ); END IF; BEGIN EXECUTE q; EXCEPTION WHEN duplicate_table THEN RAISE NOTICE 'Partition % already exists.', _partition_name; WHEN others THEN GET STACKED DIAGNOSTICS err_context = PG_EXCEPTION_CONTEXT; RAISE INFO 'Error Name:%',SQLERRM; RAISE INFO 'Error State:%', SQLSTATE; RAISE INFO 'Error Context:%', err_context; END; PERFORM maintain_partitions(_partition_name); PERFORM update_partition_stats_q(_partition_name, true); REFRESH MATERIALIZED VIEW partitions; REFRESH MATERIALIZED VIEW partition_steps; RETURN _partition_name; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION repartition(_collection text, _partition_trunc text, triggered boolean DEFAULT FALSE) RETURNS text AS $$ DECLARE c RECORD; q text; from_trunc text; BEGIN SELECT * INTO c FROM pgstac.collections WHERE id=_collection; IF NOT FOUND THEN RAISE EXCEPTION 'Collection % does not exist', _collection USING ERRCODE = 'foreign_key_violation', HINT = 'Make sure collection exists before adding items'; END IF; IF triggered THEN RAISE NOTICE 'Converting % to % partitioning via Trigger', _collection, _partition_trunc; ELSE RAISE NOTICE 'Converting % from using % to % partitioning', _collection, c.partition_trunc, _partition_trunc; IF c.partition_trunc IS NOT DISTINCT FROM _partition_trunc THEN RAISE NOTICE 'Collection % already set to use partition by %', _collection, _partition_trunc; RETURN _collection; END IF; END IF; IF EXISTS (SELECT 1 FROM partitions_view WHERE collection=_collection LIMIT 1) THEN EXECUTE format( $q$ CREATE TEMP TABLE changepartitionstaging ON COMMIT DROP AS SELECT * FROM %I; DROP TABLE IF EXISTS %I CASCADE; WITH p AS ( SELECT collection, CASE WHEN %L IS NULL THEN '-infinity'::timestamptz ELSE date_trunc(%L, datetime) END as d, tstzrange(min(datetime),max(datetime),'[]') as dtrange, tstzrange(min(end_datetime),max(end_datetime),'[]') as edtrange FROM changepartitionstaging GROUP BY 1,2 ) SELECT check_partition(collection, dtrange, edtrange) FROM p; INSERT INTO items SELECT * FROM changepartitionstaging; DROP TABLE changepartitionstaging; $q$, concat('_items_', c.key), concat('_items_', c.key), c.partition_trunc, c.partition_trunc ); END IF; RETURN _collection; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION collections_trigger_func() RETURNS TRIGGER AS $$ DECLARE q text; partition_name text := format('_items_%s', NEW.key); partition_exists boolean := false; partition_empty boolean := true; err_context text; loadtemp boolean := FALSE; BEGIN RAISE NOTICE 'Collection Trigger. % %', NEW.id, NEW.key; IF TG_OP = 'UPDATE' AND NEW.partition_trunc IS DISTINCT FROM OLD.partition_trunc THEN PERFORM repartition(NEW.id, NEW.partition_trunc, TRUE); END IF; RETURN NEW; END; $$ LANGUAGE PLPGSQL; CREATE TRIGGER collections_trigger AFTER INSERT OR UPDATE ON collections FOR EACH ROW EXECUTE FUNCTION collections_trigger_func(); ================================================ FILE: src/pgstac/sql/004_search.sql ================================================ CREATE OR REPLACE FUNCTION chunker( IN _where text, OUT s timestamptz, OUT e timestamptz ) RETURNS SETOF RECORD AS $$ DECLARE explain jsonb; BEGIN IF _where IS NULL THEN _where := ' TRUE '; END IF; EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s;', _where) INTO explain; RAISE DEBUG 'EXPLAIN: %', explain; RETURN QUERY WITH t AS ( SELECT j->>0 as p FROM jsonb_path_query( explain, 'strict $.**."Relation Name" ? (@ != null)' ) j ), parts AS ( SELECT sdate, edate FROM t JOIN partition_steps ON (t.p = name) ), times AS ( SELECT sdate FROM parts UNION SELECT edate FROM parts ), uniq AS ( SELECT DISTINCT sdate FROM times ORDER BY sdate ), last AS ( SELECT sdate, lead(sdate, 1) over () as edate FROM uniq ) SELECT sdate, edate FROM last WHERE edate IS NOT NULL; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION partition_queries( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL ) RETURNS SETOF text AS $$ DECLARE query text; sdate timestamptz; edate timestamptz; BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RETURN NEXT format($q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s $q$, sdate, edate, _where, _orderby ); END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s $q$, _where, _orderby ); RETURN NEXT query; RETURN; END IF; RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE OR REPLACE FUNCTION partition_query_view( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN _limit int DEFAULT 10 ) RETURNS text AS $$ WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total LIMIT %s $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ), _limit )) ELSE NULL END FROM p; $$ LANGUAGE SQL IMMUTABLE; CREATE OR REPLACE FUNCTION q_to_tsquery (jinput jsonb) RETURNS tsquery AS $$ DECLARE input text; processed_text text; temp_text text; quote_array text[]; placeholder text := '@QUOTE@'; BEGIN IF jsonb_typeof(jinput) = 'string' THEN input := jinput->>0; ELSIF jsonb_typeof(jinput) = 'array' THEN input := array_to_string( array(select jsonb_array_elements_text(jinput)), ' OR ' ); ELSE RAISE EXCEPTION 'Input must be a string or an array of strings.'; END IF; -- Extract all quoted phrases and store in array quote_array := regexp_matches(input, '"[^"]*"', 'g'); -- Replace each quoted part with a unique placeholder if there are any quoted phrases IF array_length(quote_array, 1) IS NOT NULL THEN processed_text := input; FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, quote_array[i], placeholder || i || placeholder); END LOOP; ELSE processed_text := input; END IF; -- Replace non-quoted text using regular expressions -- , -> | processed_text := regexp_replace(processed_text, ',(?=(?:[^"]*"[^"]*")*[^"]*$)', ' | ', 'g'); -- and -> & processed_text := regexp_replace(processed_text, '\s+AND\s+', ' & ', 'gi'); -- or -> | processed_text := regexp_replace(processed_text, '\s+OR\s+', ' | ', 'gi'); -- + -> processed_text := regexp_replace(processed_text, '^\s*\+([a-zA-Z0-9_]+)', '\1', 'g'); -- +term at start processed_text := regexp_replace(processed_text, '\s*\+([a-zA-Z0-9_]+)', ' & \1', 'g'); -- +term elsewhere -- - -> ! processed_text := regexp_replace(processed_text, '^\s*\-([a-zA-Z0-9_]+)', '! \1', 'g'); -- -term at start processed_text := regexp_replace(processed_text, '\s*\-([a-zA-Z0-9_]+)', ' & ! \1', 'g'); -- -term elsewhere -- terms separated with spaces are assumed to represent adjacent terms. loop through these -- occurrences and replace them with the adjacency operator (<->) LOOP temp_text := regexp_replace(processed_text, '([a-zA-Z0-9_]+)\s+([a-zA-Z0-9_]+)(?!\s*[&|<>])', '\1 <-> \2', 'g'); IF temp_text = processed_text THEN EXIT; -- No more replacements were made END IF; processed_text := temp_text; END LOOP; -- Replace placeholders back with quoted phrases if there were any IF array_length(quote_array, 1) IS NOT NULL THEN FOR i IN array_lower(quote_array, 1) .. array_upper(quote_array, 1) LOOP processed_text := replace(processed_text, placeholder || i || placeholder, '''' || substring(quote_array[i] from 2 for length(quote_array[i]) - 2) || ''''); END LOOP; END IF; -- Print processed_text to the console for debugging purposes RAISE NOTICE 'processed_text: %', processed_text; RETURN to_tsquery('english', processed_text); END; $$ LANGUAGE plpgsql; CREATE OR REPLACE FUNCTION stac_search_to_where(j jsonb) RETURNS text AS $$ DECLARE where_segments text[]; _where text; dtrange tstzrange; collections text[]; geom geometry; sdate timestamptz; edate timestamptz; filterlang text; filter jsonb := j->'filter'; ft_query tsquery; BEGIN IF j ? 'ids' THEN where_segments := where_segments || format('id = ANY (%L) ', to_text_array(j->'ids')); END IF; IF j ? 'collections' THEN collections := to_text_array(j->'collections'); where_segments := where_segments || format('collection = ANY (%L) ', collections); END IF; IF j ? 'datetime' THEN dtrange := parse_dtrange(j->'datetime'); sdate := lower(dtrange); edate := upper(dtrange); where_segments := where_segments || format(' datetime <= %L::timestamptz AND end_datetime >= %L::timestamptz ', edate, sdate ); END IF; IF j ? 'q' THEN ft_query := q_to_tsquery(j->'q'); where_segments := where_segments || format( $quote$ ( to_tsvector('english', content->'properties'->>'description') || to_tsvector('english', coalesce(content->'properties'->>'title', '')) || to_tsvector('english', coalesce(content->'properties'->>'keywords', '')) ) @@ %L $quote$, ft_query ); END IF; geom := stac_geom(j); IF geom IS NOT NULL THEN where_segments := where_segments || format('st_intersects(geometry, %L)',geom); END IF; filterlang := COALESCE( j->>'filter-lang', get_setting('default_filter_lang', j->'conf') ); IF NOT filter @? '$.**.op' THEN filterlang := 'cql-json'; END IF; IF filterlang NOT IN ('cql-json','cql2-json') AND j ? 'filter' THEN RAISE EXCEPTION '% is not a supported filter-lang. Please use cql-json or cql2-json.', filterlang; END IF; IF j ? 'query' AND j ? 'filter' THEN RAISE EXCEPTION 'Can only use either query or filter at one time.'; END IF; IF j ? 'query' THEN filter := query_to_cql2(j->'query'); ELSIF filterlang = 'cql-json' THEN filter := cql1_to_cql2(filter); END IF; RAISE NOTICE 'FILTER: %', filter; where_segments := where_segments || cql2_query(filter); IF cardinality(where_segments) < 1 THEN RETURN ' TRUE '; END IF; _where := array_to_string(array_remove(where_segments, NULL), ' AND '); IF _where IS NULL OR BTRIM(_where) = '' THEN RETURN ' TRUE '; END IF; RETURN _where; END; $$ LANGUAGE PLPGSQL STABLE; CREATE OR REPLACE FUNCTION parse_sort_dir(_dir text, reverse boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN NOT reverse THEN d WHEN d = 'ASC' THEN 'DESC' WHEN d = 'DESC' THEN 'ASC' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_dir_to_op(_dir text, prev boolean default false) RETURNS text AS $$ WITH t AS ( SELECT COALESCE(upper(_dir), 'ASC') as d ) SELECT CASE WHEN d = 'ASC' AND prev THEN '<=' WHEN d = 'DESC' AND prev THEN '>=' WHEN d = 'ASC' THEN '>=' WHEN d = 'DESC' THEN '<=' END FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION sort_sqlorderby( _search jsonb DEFAULT NULL, reverse boolean DEFAULT FALSE ) RETURNS text AS $$ WITH sortby AS ( SELECT coalesce(_search->'sortby','[{"field":"datetime", "direction":"desc"}]') as sort ), withid AS ( SELECT CASE WHEN sort @? '$[*] ? (@.field == "id")' THEN sort ELSE sort || '[{"field":"id", "direction":"desc"}]'::jsonb END as sort FROM sortby ), withid_rows AS ( SELECT jsonb_array_elements(sort) as value FROM withid ),sorts AS ( SELECT coalesce( (queryable(value->>'field')).expression ) as key, parse_sort_dir(value->>'direction', reverse) as dir FROM withid_rows ) SELECT array_to_string( array_agg(concat(key, ' ', dir)), ', ' ) FROM sorts; $$ LANGUAGE SQL; CREATE OR REPLACE FUNCTION get_sort_dir(sort_item jsonb) RETURNS text AS $$ SELECT CASE WHEN sort_item->>'direction' ILIKE 'desc%' THEN 'DESC' ELSE 'ASC' END; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION get_token_val_str( _field text, _item items ) RETURNS text AS $$ DECLARE q text; literal text; BEGIN q := format($q$ SELECT quote_literal(%s) FROM (SELECT $1.*) as r;$q$, _field); EXECUTE q INTO literal USING _item; RETURN literal; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION get_token_record(IN _token text, OUT prev BOOLEAN, OUT item items) RETURNS RECORD AS $$ DECLARE _itemid text := _token; _collectionid text; BEGIN IF _token IS NULL THEN RETURN; END IF; RAISE NOTICE 'Looking for token: %', _token; prev := FALSE; IF _token ILIKE 'prev:%' THEN _itemid := replace(_token, 'prev:',''); prev := TRUE; ELSIF _token ILIKE 'next:%' THEN _itemid := replace(_token, 'next:', ''); END IF; SELECT id INTO _collectionid FROM collections WHERE _itemid LIKE concat(id,':%'); IF FOUND THEN _itemid := replace(_itemid, concat(_collectionid,':'), ''); SELECT * INTO item FROM items WHERE id=_itemid AND collection=_collectionid; ELSE SELECT * INTO item FROM items WHERE id=_itemid; END IF; IF item IS NULL THEN RAISE EXCEPTION 'Could not find item using token: % item: % collection: %', _token, _itemid, _collectionid; END IF; RETURN; END; $$ LANGUAGE PLPGSQL STABLE STRICT; CREATE OR REPLACE FUNCTION get_token_filter( _sortby jsonb DEFAULT '[{"field":"datetime","direction":"desc"}]'::jsonb, token_item items DEFAULT NULL, prev boolean DEFAULT FALSE, inclusive boolean DEFAULT FALSE ) RETURNS text AS $$ DECLARE ltop text := '<'; gtop text := '>'; dir text; sort record; orfilter text := ''; orfilters text[] := '{}'::text[]; andfilters text[] := '{}'::text[]; output text; token_where text; BEGIN IF _sortby IS NULL OR _sortby = '[]'::jsonb THEN _sortby := '[{"field":"datetime","direction":"desc"}]'::jsonb; END IF; _sortby := _sortby || jsonb_build_object('field','id','direction',_sortby->0->>'direction'); RAISE NOTICE 'Getting Token Filter. % %', _sortby, token_item; IF inclusive THEN orfilters := orfilters || format('( id=%L AND collection=%L )' , token_item.id, token_item.collection); END IF; FOR sort IN WITH s1 AS ( SELECT _row, (queryable(value->>'field')).expression as _field, (value->>'field' = 'id') as _isid, get_sort_dir(value) as _dir FROM jsonb_array_elements(_sortby) WITH ORDINALITY AS t(value, _row) ) SELECT _row, _field, _dir, get_token_val_str(_field, token_item) as _val FROM s1 WHERE _row <= (SELECT min(_row) FROM s1 WHERE _isid) LOOP orfilter := NULL; RAISE NOTICE 'SORT: %', sort; IF sort._val IS NOT NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, ltop, sort._val, sort._val ); ELSIF sort._val IS NULL AND ((prev AND sort._dir = 'ASC') OR (NOT prev AND sort._dir = 'DESC')) THEN RAISE NOTICE '< but null'; orfilter := format('%s IS NOT NULL', sort._field); ELSIF sort._val IS NULL THEN RAISE NOTICE '> but null'; ELSE orfilter := format($f$( (%s %s %s) OR (%s IS NULL) )$f$, sort._field, gtop, sort._val, sort._field ); END IF; RAISE NOTICE 'ORFILTER: %', orfilter; IF orfilter IS NOT NULL THEN IF sort._row = 1 THEN orfilters := orfilters || orfilter; ELSE orfilters := orfilters || format('(%s AND %s)', array_to_string(andfilters, ' AND '), orfilter); END IF; END IF; IF sort._val IS NOT NULL THEN andfilters := andfilters || format('%s = %s', sort._field, sort._val); ELSE andfilters := andfilters || format('%s IS NULL', sort._field); END IF; END LOOP; output := array_to_string(orfilters, ' OR '); token_where := concat('(',coalesce(output,'true'),')'); IF trim(token_where) = '' THEN token_where := NULL; END IF; RAISE NOTICE 'TOKEN_WHERE: %',token_where; RETURN token_where; END; $$ LANGUAGE PLPGSQL SET transform_null_equals TO TRUE ; CREATE OR REPLACE FUNCTION search_hash(jsonb, jsonb) RETURNS text AS $$ SELECT md5(concat(($1 - '{token,limit,context,includes,excludes}'::text[])::text,$2::text)); $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; DROP FUNCTION IF EXISTS search_tohash(jsonb); CREATE TABLE IF NOT EXISTS searches( hash text GENERATED ALWAYS AS (search_hash(search, metadata)) STORED PRIMARY KEY, search jsonb NOT NULL, _where text, orderby text, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, metadata jsonb DEFAULT '{}'::jsonb NOT NULL ); CREATE TABLE IF NOT EXISTS search_wheres( id bigint generated always as identity primary key, _where text NOT NULL, lastused timestamptz DEFAULT now(), usecount bigint DEFAULT 0, statslastupdated timestamptz, estimated_count bigint, estimated_cost float, time_to_estimate float, total_count bigint, time_to_count float, partitions text[] ); CREATE INDEX IF NOT EXISTS search_wheres_partitions ON search_wheres USING GIN (partitions); CREATE UNIQUE INDEX IF NOT EXISTS search_wheres_where ON search_wheres ((md5(_where))); CREATE OR REPLACE FUNCTION where_stats( inwhere text, updatestats boolean default false, conf jsonb default null ) RETURNS search_wheres AS $$ DECLARE t timestamptz; i interval; explain_json jsonb; partitions text[]; sw search_wheres%ROWTYPE; inwhere_hash text := md5(inwhere); _context text := lower(context(conf)); _stats_ttl interval := context_stats_ttl(conf); _estimated_cost_threshold float := context_estimated_cost(conf); _estimated_count_threshold int := context_estimated_count(conf); ro bool := pgstac.readonly(conf); BEGIN -- If updatestats is true then set ttl to 0 IF updatestats THEN RAISE DEBUG 'Updatestats set to TRUE, setting TTL to 0'; _stats_ttl := '0'::interval; END IF; -- If we don't need to calculate context, just return IF _context = 'off' THEN sw._where = inwhere; RETURN sw; END IF; -- Get any stats that we have. IF NOT ro THEN -- If there is a lock where another process is -- updating the stats, wait so that we don't end up calculating a bunch of times. SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash FOR UPDATE; ELSE SELECT * INTO sw FROM search_wheres WHERE md5(_where)=inwhere_hash; END IF; -- If there is a cached row, figure out if we need to update IF sw IS NOT NULL AND sw.statslastupdated IS NOT NULL AND sw.total_count IS NOT NULL AND now() - sw.statslastupdated <= _stats_ttl THEN -- we have a cached row with data that is within our ttl RAISE DEBUG 'Stats present in table and lastupdated within ttl: %', sw; IF NOT ro THEN RAISE DEBUG 'Updating search_wheres only bumping lastused and usecount'; UPDATE search_wheres SET lastused = now(), usecount = search_wheres.usecount + 1 WHERE md5(_where) = inwhere_hash RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning cached counts. %', sw; RETURN sw; END IF; -- Calculate estimated cost and rows -- Use explain to get estimated count/cost IF sw.estimated_count IS NULL OR sw.estimated_cost IS NULL THEN RAISE DEBUG 'Calculating estimated stats'; t := clock_timestamp(); EXECUTE format('EXPLAIN (format json) SELECT 1 FROM items WHERE %s', inwhere) INTO explain_json; RAISE DEBUG 'Time for just the explain: %', clock_timestamp() - t; i := clock_timestamp() - t; sw.estimated_count := explain_json->0->'Plan'->'Plan Rows'; sw.estimated_cost := explain_json->0->'Plan'->'Total Cost'; sw.time_to_estimate := extract(epoch from i); END IF; RAISE DEBUG 'ESTIMATED_COUNT: %, THRESHOLD %', sw.estimated_count, _estimated_count_threshold; RAISE DEBUG 'ESTIMATED_COST: %, THRESHOLD %', sw.estimated_cost, _estimated_cost_threshold; -- If context is set to auto and the costs are within the threshold return the estimated costs IF _context = 'auto' AND sw.estimated_count >= _estimated_count_threshold AND sw.estimated_cost >= _estimated_cost_threshold THEN IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, null, null ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Estimates are within thresholds, returning estimates. %', sw; RETURN sw; END IF; -- Calculate Actual Count t := clock_timestamp(); RAISE NOTICE 'Calculating actual count...'; EXECUTE format( 'SELECT count(*) FROM items WHERE %s', inwhere ) INTO sw.total_count; i := clock_timestamp() - t; RAISE NOTICE 'Actual Count: % -- %', sw.total_count, i; sw.time_to_count := extract(epoch FROM i); IF NOT ro THEN INSERT INTO search_wheres ( _where, lastused, usecount, statslastupdated, estimated_count, estimated_cost, time_to_estimate, total_count, time_to_count ) VALUES ( inwhere, now(), 1, now(), sw.estimated_count, sw.estimated_cost, sw.time_to_estimate, sw.total_count, sw.time_to_count ) ON CONFLICT ((md5(_where))) DO UPDATE SET lastused = EXCLUDED.lastused, usecount = search_wheres.usecount + 1, statslastupdated = EXCLUDED.statslastupdated, estimated_count = EXCLUDED.estimated_count, estimated_cost = EXCLUDED.estimated_cost, time_to_estimate = EXCLUDED.time_to_estimate, total_count = EXCLUDED.total_count, time_to_count = EXCLUDED.time_to_count RETURNING * INTO sw; END IF; RAISE DEBUG 'Returning with actual count. %', sw; RETURN sw; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_query( _search jsonb = '{}'::jsonb, updatestats boolean = false, _metadata jsonb = '{}'::jsonb ) RETURNS searches AS $$ DECLARE search searches%ROWTYPE; cached_search searches%ROWTYPE; pexplain jsonb; t timestamptz; i interval; doupdate boolean := FALSE; insertfound boolean := FALSE; ro boolean := pgstac.readonly(); found_search text; BEGIN RAISE NOTICE 'SEARCH: %', _search; -- Calculate hash, where clause, and order by statement search.search := _search; search.metadata := _metadata; search.hash := search_hash(_search, _metadata); search._where := stac_search_to_where(_search); search.orderby := sort_sqlorderby(_search); search.lastused := now(); search.usecount := 1; -- If we are in read only mode, directly return search IF ro THEN RETURN search; END IF; RAISE NOTICE 'Updating Statistics for search: %s', search; -- Update statistics for times used and and when last used -- If the entry is locked, rather than waiting, skip updating the stats INSERT INTO searches (search, lastused, usecount, metadata) VALUES (search.search, now(), 1, search.metadata) ON CONFLICT DO NOTHING RETURNING * INTO cached_search ; IF NOT FOUND OR cached_search IS NULL THEN UPDATE searches SET lastused = now(), usecount = searches.usecount + 1 WHERE hash = ( SELECT hash FROM searches WHERE hash=search.hash FOR UPDATE SKIP LOCKED ) RETURNING * INTO cached_search ; END IF; IF cached_search IS NOT NULL THEN cached_search._where = search._where; cached_search.orderby = search.orderby; RETURN cached_search; END IF; RETURN search; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search_fromhash( _hash text ) RETURNS searches AS $$ SELECT * FROM search_query((SELECT search FROM searches WHERE hash=_hash LIMIT 1)); $$ LANGUAGE SQL STRICT; CREATE OR REPLACE FUNCTION search_rows( IN _where text DEFAULT 'TRUE', IN _orderby text DEFAULT 'datetime DESC, id DESC', IN partitions text[] DEFAULT NULL, IN _limit int DEFAULT 10 ) RETURNS SETOF items AS $$ DECLARE base_query text; query text; sdate timestamptz; edate timestamptz; n int; records_left int := _limit; timer timestamptz := clock_timestamp(); full_timer timestamptz := clock_timestamp(); BEGIN IF _where IS NULL OR trim(_where) = '' THEN _where = ' TRUE '; END IF; RAISE NOTICE 'Getting chunks for % %', _where, _orderby; base_query := $q$ SELECT * FROM items WHERE datetime >= %L AND datetime < %L AND (%s) ORDER BY %s LIMIT %L $q$; IF _orderby ILIKE 'datetime d%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 DESC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSIF _orderby ILIKE 'datetime a%' THEN FOR sdate, edate IN SELECT * FROM chunker(_where) ORDER BY 1 ASC LOOP RAISE NOTICE 'Running Query for % to %. %', sdate, edate, age_ms(full_timer); query := format( base_query, sdate, edate, _where, _orderby, records_left ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; GET DIAGNOSTICS n = ROW_COUNT; records_left := records_left - n; RAISE NOTICE 'Returned %/% Rows From % to %. % to go. Time: %ms', n, _limit, sdate, edate, records_left, age_ms(timer); timer := clock_timestamp(); IF records_left <= 0 THEN RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END IF; END LOOP; ELSE query := format($q$ SELECT * FROM items WHERE %s ORDER BY %s LIMIT %L $q$, _where, _orderby, _limit ); RAISE DEBUG 'QUERY: %', query; timer := clock_timestamp(); RETURN QUERY EXECUTE query; RAISE NOTICE 'FULL QUERY TOOK %ms', age_ms(timer); END IF; RAISE NOTICE 'SEARCH_ROWS TOOK %ms', age_ms(full_timer); RETURN; END; $$ LANGUAGE PLPGSQL SET SEARCH_PATH TO pgstac,public; CREATE UNLOGGED TABLE format_item_cache( id text, collection text, fields text, hydrated bool, output jsonb, lastused timestamptz DEFAULT now(), usecount int DEFAULT 1, timetoformat float, PRIMARY KEY (collection, id, fields, hydrated) ); CREATE INDEX ON format_item_cache (lastused); CREATE OR REPLACE FUNCTION format_item(_item items, _fields jsonb DEFAULT '{}', _hydrated bool DEFAULT TRUE) RETURNS jsonb AS $$ DECLARE cache bool := get_setting_bool('format_cache'); _output jsonb := null; t timestamptz := clock_timestamp(); BEGIN IF cache THEN SELECT output INTO _output FROM format_item_cache WHERE id=_item.id AND collection=_item.collection AND fields=_fields::text AND hydrated=_hydrated; END IF; IF _output IS NULL THEN IF _hydrated THEN _output := content_hydrate(_item, _fields); ELSE _output := content_nonhydrated(_item, _fields); END IF; END IF; IF cache THEN INSERT INTO format_item_cache (id, collection, fields, hydrated, output, timetoformat) VALUES (_item.id, _item.collection, _fields::text, _hydrated, _output, age_ms(t)) ON CONFLICT(collection, id, fields, hydrated) DO UPDATE SET lastused=now(), usecount = format_item_cache.usecount + 1 ; END IF; RETURN _output; END; $$ LANGUAGE PLPGSQL SECURITY DEFINER; CREATE OR REPLACE FUNCTION search(_search jsonb = '{}'::jsonb) RETURNS jsonb AS $$ DECLARE searches searches%ROWTYPE; _where text; orderby text; search_where search_wheres%ROWTYPE; total_count bigint; token record; token_prev boolean; token_item items%ROWTYPE; token_where text; full_where text; init_ts timestamptz := clock_timestamp(); timer timestamptz := clock_timestamp(); hydrate bool := NOT (_search->'conf'->>'nohydrate' IS NOT NULL AND (_search->'conf'->>'nohydrate')::boolean = true); prev text; next text; context jsonb; collection jsonb; out_records jsonb; out_len int; _limit int := coalesce((_search->>'limit')::int, 10); _querylimit int; _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); has_prev boolean := FALSE; has_next boolean := FALSE; links jsonb := '[]'::jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/')); BEGIN searches := search_query(_search); _where := searches._where; orderby := searches.orderby; search_where := where_stats(_where); total_count := coalesce(search_where.total_count, search_where.estimated_count); RAISE NOTICE 'SEARCH:TOKEN: %', _search->>'token'; token := get_token_record(_search->>'token'); RAISE NOTICE '***TOKEN: %', token; _querylimit := _limit + 1; IF token IS NOT NULL THEN token_prev := token.prev; token_item := token.item; token_where := get_token_filter(_search->'sortby', token_item, token_prev, FALSE); RAISE DEBUG 'TOKEN_WHERE: % (%ms from search start)', token_where, age_ms(timer); IF token_prev THEN -- if we are using a prev token, we know has_next is true RAISE DEBUG 'There is a previous token, so automatically setting has_next to true'; has_next := TRUE; orderby := sort_sqlorderby(_search, TRUE); ELSE RAISE DEBUG 'There is a next token, so automatically setting has_prev to true'; has_prev := TRUE; END IF; ELSE -- if there was no token, we know there is no prev RAISE DEBUG 'There is no token, so we know there is no prev. setting has_prev to false'; has_prev := FALSE; END IF; full_where := concat_ws(' AND ', _where, token_where); RAISE NOTICE 'FULL WHERE CLAUSE: %', full_where; RAISE NOTICE 'Time to get counts and build query %', age_ms(timer); timer := clock_timestamp(); IF hydrate THEN RAISE NOTICE 'Getting hydrated data.'; ELSE RAISE NOTICE 'Getting non-hydrated data.'; END IF; RAISE NOTICE 'CACHE SET TO %', get_setting_bool('format_cache'); RAISE NOTICE 'Time to set hydration/formatting %', age_ms(timer); timer := clock_timestamp(); SELECT jsonb_agg(format_item(i, _fields, hydrate)) INTO out_records FROM search_rows( full_where, orderby, search_where.partitions, _querylimit ) as i; RAISE NOTICE 'Time to fetch rows %', age_ms(timer); timer := clock_timestamp(); IF token_prev THEN out_records := flip_jsonb_array(out_records); END IF; RAISE NOTICE 'Query returned % records.', jsonb_array_length(out_records); RAISE DEBUG 'TOKEN: % %', token_item.id, token_item.collection; RAISE DEBUG 'RECORD_1: % %', out_records->0->>'id', out_records->0->>'collection'; RAISE DEBUG 'RECORD-1: % %', out_records->-1->>'id', out_records->-1->>'collection'; -- REMOVE records that were from our token IF out_records->0->>'id' = token_item.id AND out_records->0->>'collection' = token_item.collection THEN out_records := out_records - 0; ELSIF out_records->-1->>'id' = token_item.id AND out_records->-1->>'collection' = token_item.collection THEN out_records := out_records - -1; END IF; out_len := jsonb_array_length(out_records); IF out_len = _limit + 1 THEN IF token_prev THEN has_prev := TRUE; out_records := out_records - 0; ELSE has_next := TRUE; out_records := out_records - -1; END IF; END IF; links := links || jsonb_build_object( 'rel', 'root', 'type', 'application/json', 'href', base_url ) || jsonb_build_object( 'rel', 'self', 'type', 'application/json', 'href', concat(base_url, '/search') ); IF has_next THEN next := concat(out_records->-1->>'collection', ':', out_records->-1->>'id'); RAISE NOTICE 'HAS NEXT | %', next; links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=next:', next) ); END IF; IF has_prev THEN prev := concat(out_records->0->>'collection', ':', out_records->0->>'id'); RAISE NOTICE 'HAS PREV | %', prev; links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/geo+json', 'method', 'GET', 'href', concat(base_url, '/search?token=prev:', prev) ); END IF; RAISE NOTICE 'Time to get prev/next %', age_ms(timer); timer := clock_timestamp(); collection := jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb), 'links', links ); IF context(_search->'conf') != 'off' THEN collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberMatched', total_count, 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); ELSE collection := collection || jsonb_strip_nulls(jsonb_build_object( 'numberReturned', coalesce(jsonb_array_length(out_records), 0) )); END IF; IF get_setting_bool('timing', _search->'conf') THEN collection = collection || jsonb_build_object('timing', age_ms(init_ts)); END IF; RAISE NOTICE 'Time to build final json %', age_ms(timer); timer := clock_timestamp(); RAISE NOTICE 'Total Time: %', age_ms(current_timestamp); RAISE NOTICE 'RETURNING % records. NEXT: %. PREV: %', collection->>'numberReturned', collection->>'next', collection->>'prev'; RETURN collection; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION search_cursor(_search jsonb = '{}'::jsonb) RETURNS refcursor AS $$ DECLARE curs refcursor; searches searches%ROWTYPE; _where text; _orderby text; q text; BEGIN searches := search_query(_search); _where := searches._where; _orderby := searches.orderby; OPEN curs FOR WITH p AS ( SELECT * FROM partition_queries(_where, _orderby) p ) SELECT CASE WHEN EXISTS (SELECT 1 FROM p) THEN (SELECT format($q$ SELECT * FROM ( %s ) total $q$, string_agg( format($q$ SELECT * FROM ( %s ) AS sub $q$, p), ' UNION ALL ' ) )) ELSE NULL END FROM p; RETURN curs; END; $$ LANGUAGE PLPGSQL; ================================================ FILE: src/pgstac/sql/004a_collectionsearch.sql ================================================ CREATE OR REPLACE VIEW collections_asitems AS SELECT id, geometry, 'collections' AS collection, datetime, end_datetime, jsonb_build_object( 'properties', content - '{links,assets,stac_version,stac_extensions}', 'links', content->'links', 'assets', content->'assets', 'stac_version', content->'stac_version', 'stac_extensions', content->'stac_extensions' ) AS content, content as collectionjson FROM collections; CREATE OR REPLACE FUNCTION collection_search_matched( IN _search jsonb DEFAULT '{}'::jsonb, OUT matched bigint ) RETURNS bigint AS $$ DECLARE _where text := stac_search_to_where(_search); BEGIN EXECUTE format( $query$ SELECT count(*) FROM collections_asitems WHERE %s ; $query$, _where ) INTO matched; RETURN; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION collection_search_rows( _search jsonb DEFAULT '{}'::jsonb ) RETURNS SETOF jsonb AS $$ DECLARE _where text := stac_search_to_where(_search); _limit int := coalesce((_search->>'limit')::int, 10); _fields jsonb := coalesce(_search->'fields', '{}'::jsonb); _orderby text; _offset int := COALESCE((_search->>'offset')::int, 0); BEGIN _orderby := sort_sqlorderby( jsonb_build_object( 'sortby', coalesce( _search->'sortby', '[{"field": "id", "direction": "asc"}]'::jsonb ) ) ); RETURN QUERY EXECUTE format( $query$ SELECT jsonb_fields(collectionjson, %L) as c FROM collections_asitems WHERE %s ORDER BY %s LIMIT %L OFFSET %L ; $query$, _fields, _where, _orderby, _limit, _offset ); END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_search( _search jsonb DEFAULT '{}'::jsonb ) RETURNS jsonb AS $$ DECLARE out_records jsonb; number_matched bigint := collection_search_matched(_search); number_returned bigint; _limit int := coalesce((_search->>'limit')::float::int, 10); _offset int := coalesce((_search->>'offset')::float::int, 0); links jsonb := '[]'; ret jsonb; base_url text:= concat(rtrim(base_url(_search->'conf'),'/'), '/collections'); prevoffset int; nextoffset int; BEGIN SELECT coalesce(jsonb_agg(c), '[]') INTO out_records FROM collection_search_rows(_search) c; number_returned := jsonb_array_length(out_records); RAISE DEBUG 'nm: %, nr: %, l:%, o:%', number_matched, number_returned, _limit, _offset; IF _limit <= number_matched AND number_matched > 0 THEN --need to have paging links nextoffset := least(_offset + _limit, number_matched - 1); prevoffset := greatest(_offset - _limit, 0); IF _offset > 0 THEN links := links || jsonb_build_object( 'rel', 'prev', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', prevoffset), 'merge', TRUE ); END IF; IF (_offset + _limit < number_matched) THEN links := links || jsonb_build_object( 'rel', 'next', 'type', 'application/json', 'method', 'GET' , 'href', base_url, 'body', jsonb_build_object('offset', nextoffset), 'merge', TRUE ); END IF; END IF; ret := jsonb_build_object( 'collections', out_records, 'numberMatched', number_matched, 'numberReturned', number_returned, 'links', links ); RETURN ret; END; $$ LANGUAGE PLPGSQL STABLE PARALLEL SAFE; ================================================ FILE: src/pgstac/sql/005_tileutils.sql ================================================ SET SEARCH_PATH TO pgstac, public; CREATE OR REPLACE FUNCTION tileenvelope(zoom int, x int, y int) RETURNS geometry AS $$ WITH t AS ( SELECT 20037508.3427892 as merc_max, -20037508.3427892 as merc_min, (2 * 20037508.3427892) / (2 ^ zoom) as tile_size ) SELECT st_makeenvelope( merc_min + (tile_size * x), merc_max - (tile_size * (y + 1)), merc_min + (tile_size * (x + 1)), merc_max - (tile_size * y), 3857 ) FROM t; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE;DROP FUNCTION IF EXISTS mercgrid; CREATE OR REPLACE FUNCTION ftime() RETURNS interval as $$ SELECT age(clock_timestamp(), transaction_timestamp()); $$ LANGUAGE SQL; ================================================ FILE: src/pgstac/sql/006_tilesearch.sql ================================================ SET SEARCH_PATH to pgstac, public; DROP FUNCTION IF EXISTS geometrysearch; CREATE OR REPLACE FUNCTION geometrysearch( IN geom geometry, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, -- Return as soon as the passed in geometry is full covered IN skipcovered boolean DEFAULT TRUE -- Skip any items that would show up completely under the previous items ) RETURNS jsonb AS $$ DECLARE search searches%ROWTYPE; curs refcursor; _where text; query text; iter_record items%ROWTYPE; out_records jsonb := '{}'::jsonb[]; exit_flag boolean := FALSE; counter int := 1; scancounter int := 1; remaining_limit int := _scanlimit; tilearea float; unionedgeom geometry; clippedgeom geometry; unionedgeom_area float := 0; prev_area float := 0; excludes text[]; includes text[]; BEGIN DROP TABLE IF EXISTS pgstac_results; CREATE TEMP TABLE pgstac_results (content jsonb) ON COMMIT DROP; -- If the passed in geometry is not an area set exitwhenfull and skipcovered to false IF ST_GeometryType(geom) !~* 'polygon' THEN RAISE NOTICE 'GEOMETRY IS NOT AN AREA'; skipcovered = FALSE; exitwhenfull = FALSE; END IF; -- If skipcovered is true then you will always want to exit when the passed in geometry is full IF skipcovered THEN exitwhenfull := TRUE; END IF; search := search_fromhash(queryhash); IF search IS NULL THEN RAISE EXCEPTION 'Search with Query Hash % Not Found', queryhash; END IF; tilearea := st_area(geom); _where := format('%s AND st_intersects(geometry, %L::geometry)', search._where, geom); FOR query IN SELECT * FROM partition_queries(_where, search.orderby) LOOP query := format('%s LIMIT %L', query, remaining_limit); RAISE NOTICE '%', query; OPEN curs FOR EXECUTE query; LOOP FETCH curs INTO iter_record; EXIT WHEN NOT FOUND; IF exitwhenfull OR skipcovered THEN -- If we are not using exitwhenfull or skipcovered, we do not need to do expensive geometry operations clippedgeom := st_intersection(geom, iter_record.geometry); IF unionedgeom IS NULL THEN unionedgeom := clippedgeom; ELSE unionedgeom := st_union(unionedgeom, clippedgeom); END IF; unionedgeom_area := st_area(unionedgeom); IF skipcovered AND prev_area = unionedgeom_area THEN scancounter := scancounter + 1; CONTINUE; END IF; prev_area := unionedgeom_area; RAISE NOTICE '% % % %', unionedgeom_area/tilearea, counter, scancounter, ftime(); END IF; RAISE NOTICE '% %', iter_record, content_hydrate(iter_record, fields); INSERT INTO pgstac_results (content) VALUES (content_hydrate(iter_record, fields)); IF counter >= _limit OR scancounter > _scanlimit OR ftime() > _timelimit OR (exitwhenfull AND unionedgeom_area >= tilearea) THEN exit_flag := TRUE; EXIT; END IF; counter := counter + 1; scancounter := scancounter + 1; END LOOP; CLOSE curs; EXIT WHEN exit_flag; remaining_limit := _scanlimit - scancounter; END LOOP; SELECT jsonb_agg(content) INTO out_records FROM pgstac_results WHERE content IS NOT NULL; RETURN jsonb_build_object( 'type', 'FeatureCollection', 'features', coalesce(out_records, '[]'::jsonb) ); END; $$ LANGUAGE PLPGSQL; DROP FUNCTION IF EXISTS geojsonsearch; CREATE OR REPLACE FUNCTION geojsonsearch( IN geojson jsonb, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_geomfromgeojson(geojson), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; DROP FUNCTION IF EXISTS xyzsearch; CREATE OR REPLACE FUNCTION xyzsearch( IN _x int, IN _y int, IN _z int, IN queryhash text, IN fields jsonb DEFAULT NULL, IN _scanlimit int DEFAULT 10000, IN _limit int DEFAULT 100, IN _timelimit interval DEFAULT '5 seconds'::interval, IN exitwhenfull boolean DEFAULT TRUE, IN skipcovered boolean DEFAULT TRUE ) RETURNS jsonb AS $$ SELECT * FROM geometrysearch( st_transform(tileenvelope(_z, _x, _y), 4326), queryhash, fields, _scanlimit, _limit, _timelimit, exitwhenfull, skipcovered ); $$ LANGUAGE SQL; ================================================ FILE: src/pgstac/sql/997_maintenance.sql ================================================ CREATE OR REPLACE PROCEDURE analyze_items() AS $$ DECLARE q text; timeout_ts timestamptz; BEGIN timeout_ts := statement_timestamp() + queue_timeout(); WHILE clock_timestamp() < timeout_ts LOOP SELECT format('ANALYZE (VERBOSE, SKIP_LOCKED) %I;', relname) INTO q FROM pg_stat_user_tables WHERE relname like '_item%' AND (n_mod_since_analyze>0 OR last_analyze IS NULL) LIMIT 1; IF NOT FOUND THEN EXIT; END IF; RAISE NOTICE '%', q; EXECUTE q; COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE PROCEDURE validate_constraints() AS $$ DECLARE q text; BEGIN FOR q IN SELECT FORMAT( 'ALTER TABLE %I.%I VALIDATE CONSTRAINT %I;', nsp.nspname, cls.relname, con.conname ) FROM pg_constraint AS con JOIN pg_class AS cls ON con.conrelid = cls.oid JOIN pg_namespace AS nsp ON cls.relnamespace = nsp.oid WHERE convalidated = FALSE AND contype in ('c','f') AND nsp.nspname = 'pgstac' LOOP RAISE NOTICE '%', q; PERFORM run_or_queue(q); COMMIT; END LOOP; END; $$ LANGUAGE PLPGSQL; CREATE OR REPLACE FUNCTION collection_extent(_collection text, runupdate boolean default false) RETURNS jsonb AS $$ DECLARE geom_extent geometry; mind timestamptz; maxd timestamptz; extent jsonb; BEGIN IF runupdate THEN PERFORM update_partition_stats_q(partition) FROM partitions_view WHERE collection=_collection; END IF; SELECT min(lower(dtrange)), max(upper(edtrange)), st_extent(spatial) INTO mind, maxd, geom_extent FROM partitions_view WHERE collection=_collection; IF geom_extent IS NOT NULL AND mind IS NOT NULL AND maxd IS NOT NULL THEN extent := jsonb_build_object( 'spatial', jsonb_build_object( 'bbox', to_jsonb(array[array[st_xmin(geom_extent), st_ymin(geom_extent), st_xmax(geom_extent), st_ymax(geom_extent)]]) ), 'temporal', jsonb_build_object( 'interval', to_jsonb(array[array[mind, maxd]]) ) ); RETURN extent; END IF; RETURN NULL; END; $$ LANGUAGE PLPGSQL; ================================================ FILE: src/pgstac/sql/998_idempotent_post.sql ================================================ DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('id', '{"title": "Item ID","description": "Item identifier","$ref": "https://schemas.stacspec.org/v1.0.0/item-spec/json-schema/item.json#/definitions/core/allOf/2/properties/id"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('geometry', '{"title": "Item Geometry","description": "Item Geometry","$ref": "https://geojson.org/schema/Feature.json"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DO $$ BEGIN INSERT INTO queryables (name, definition, property_wrapper, property_index_type) VALUES ('datetime','{"description": "Datetime","type": "string","title": "Acquired","format": "date-time","pattern": "(\\+00:00|Z)$"}', null, null); EXCEPTION WHEN unique_violation THEN RAISE NOTICE '%', SQLERRM USING ERRCODE = SQLSTATE; END $$; DELETE FROM queryables a USING queryables b WHERE a.name = b.name AND a.collection_ids IS NOT DISTINCT FROM b.collection_ids AND a.id > b.id; INSERT INTO pgstac_settings (name, value) VALUES ('context', 'off'), ('context_estimated_count', '100000'), ('context_estimated_cost', '100000'), ('context_stats_ttl', '1 day'), ('default_filter_lang', 'cql2-json'), ('additional_properties', 'true'), ('use_queue', 'false'), ('queue_timeout', '10 minutes'), ('update_collection_extent', 'false'), ('format_cache', 'false'), ('readonly', 'false') ON CONFLICT DO NOTHING ; INSERT INTO cql2_ops (op, template, types) VALUES ('eq', '%s = %s', NULL), ('neq', '%s != %s', NULL), ('ne', '%s != %s', NULL), ('!=', '%s != %s', NULL), ('<>', '%s != %s', NULL), ('lt', '%s < %s', NULL), ('lte', '%s <= %s', NULL), ('gt', '%s > %s', NULL), ('gte', '%s >= %s', NULL), ('le', '%s <= %s', NULL), ('ge', '%s >= %s', NULL), ('=', '%s = %s', NULL), ('<', '%s < %s', NULL), ('<=', '%s <= %s', NULL), ('>', '%s > %s', NULL), ('>=', '%s >= %s', NULL), ('like', '%s LIKE %s', NULL), ('ilike', '%s ILIKE %s', NULL), ('+', '%s + %s', NULL), ('-', '%s - %s', NULL), ('*', '%s * %s', NULL), ('/', '%s / %s', NULL), ('not', 'NOT (%s)', NULL), ('between', '%s BETWEEN %s AND %s', NULL), ('isnull', '%s IS NULL', NULL), ('upper', 'upper(%s)', NULL), ('lower', 'lower(%s)', NULL), ('casei', 'upper(%s)', NULL), ('accenti', 'unaccent(%s)', NULL) ON CONFLICT (op) DO UPDATE SET template = EXCLUDED.template ; ALTER FUNCTION to_text COST 5000; ALTER FUNCTION to_float COST 5000; ALTER FUNCTION to_int COST 5000; ALTER FUNCTION to_tstz COST 5000; ALTER FUNCTION to_text_array COST 5000; ALTER FUNCTION update_partition_stats SECURITY DEFINER; ALTER FUNCTION partition_after_triggerfunc SECURITY DEFINER; ALTER FUNCTION drop_table_constraints SECURITY DEFINER; ALTER FUNCTION create_table_constraints SECURITY DEFINER; ALTER FUNCTION check_partition SECURITY DEFINER; ALTER FUNCTION repartition SECURITY DEFINER; ALTER FUNCTION where_stats SECURITY DEFINER; ALTER FUNCTION search_query SECURITY DEFINER; ALTER FUNCTION format_item SECURITY DEFINER; ALTER FUNCTION maintain_index SECURITY DEFINER; GRANT USAGE ON SCHEMA pgstac to pgstac_read; GRANT ALL ON SCHEMA pgstac to pgstac_ingest; GRANT ALL ON SCHEMA pgstac to pgstac_admin; -- pgstac_read role limited to using function apis GRANT EXECUTE ON FUNCTION search TO pgstac_read; GRANT EXECUTE ON FUNCTION search_query TO pgstac_read; GRANT EXECUTE ON FUNCTION item_by_id TO pgstac_read; GRANT EXECUTE ON FUNCTION get_item TO pgstac_read; GRANT SELECT ON ALL TABLES IN SCHEMA pgstac TO pgstac_read; GRANT EXECUTE ON ALL FUNCTIONS IN SCHEMA pgstac to pgstac_ingest; GRANT ALL ON ALL TABLES IN SCHEMA pgstac to pgstac_ingest; GRANT USAGE ON ALL SEQUENCES IN SCHEMA pgstac to pgstac_ingest; REVOKE ALL PRIVILEGES ON PROCEDURE run_queued_queries FROM public; GRANT ALL ON PROCEDURE run_queued_queries TO pgstac_admin; REVOKE ALL PRIVILEGES ON FUNCTION run_queued_queries_intransaction FROM public; GRANT ALL ON FUNCTION run_queued_queries_intransaction TO pgstac_admin; RESET ROLE; SET ROLE pgstac_ingest; SELECT update_partition_stats_q(partition) FROM partitions_view; ================================================ FILE: src/pgstac/sql/999_version.sql ================================================ SELECT set_version('unreleased'); ================================================ FILE: src/pgstac/tests/basic/collection_searches.sql ================================================ SET pgstac.context TO 'on'; SET pgstac."default_filter_lang" TO 'cql2-json'; WITH t AS ( SELECT row_number() over () as id, x, y FROM generate_series(-180, 170, 10) as x, generate_series(-90, 80, 10) as y ), t1 AS ( SELECT concat('testcollection_', id) as id, x as minx, y as miny, x+10 as maxx, y+10 as maxy, '2000-01-01'::timestamptz + (concat(id, ' weeks'))::interval as sdt, '2000-01-01'::timestamptz + (concat(id, ' weeks'))::interval + ('2 months')::interval as edt FROM t ) SELECT create_collection(format($q$ { "id": "%s", "type": "Collection", "title": "My Test Collection.", "description": "Description of my test collection.", "extent": { "spatial": {"bbox": [[%s, %s, %s, %s]]}, "temporal": {"interval": [[%I, %I]]} }, "stac_extensions":[] } $q$, id, minx, miny, maxx, maxy, sdt, edt )::jsonb) FROM t1; select collection_search('{"ids":["testcollection_1","testcollection_2"],"limit":10, "sortby":[{"field":"id","direction":"desc"}]}'); select collection_search('{"ids":["testcollection_1","testcollection_2"],"limit":10, "sortby":[{"field":"id","direction":"asc"}]}'); select collection_search('{"ids":["testcollection_1","testcollection_2","testcollection_3"],"limit":1, "sortby":[{"field":"id","direction":"desc"}]}'); select collection_search('{"ids":["testcollection_1","testcollection_2"],"limit":1, "offset":10, "sortby":[{"field":"datetime","direction":"desc"}]}'); select collection_search('{"filter":{"op":"eq", "args":[{"property":"title"},"My Test Collection."]},"limit":10, "sortby":[{"field":"datetime","direction":"desc"}]}'); select collection_search('{"datetime":["2012-01-01","2012-01-02"], "filter":{"op":"eq", "args":[{"property":"title"},"My Test Collection."]},"limit":10, "sortby":[{"field":"datetime","direction":"desc"}]}'); select collection_search('{"ids":["testcollection_1","testcollection_2"], "fields": {"include": ["title"]}}'); ================================================ FILE: src/pgstac/tests/basic/collection_searches.sql.out ================================================ SET pgstac.context TO 'on'; SET SET pgstac."default_filter_lang" TO 'cql2-json'; SET WITH t AS ( SELECT row_number() over () as id, x, y FROM generate_series(-180, 170, 10) as x, generate_series(-90, 80, 10) as y ), t1 AS ( SELECT concat('testcollection_', id) as id, x as minx, y as miny, x+10 as maxx, y+10 as maxy, '2000-01-01'::timestamptz + (concat(id, ' weeks'))::interval as sdt, '2000-01-01'::timestamptz + (concat(id, ' weeks'))::interval + ('2 months')::interval as edt FROM t ) SELECT create_collection(format($q$ { "id": "%s", "type": "Collection", "title": "My Test Collection.", "description": "Description of my test collection.", "extent": { "spatial": {"bbox": [[%s, %s, %s, %s]]}, "temporal": {"interval": [[%I, %I]]} }, "stac_extensions":[] } $q$, id, minx, miny, maxx, maxy, sdt, edt )::jsonb) FROM t1; select collection_search('{"ids":["testcollection_1","testcollection_2"],"limit":10, "sortby":[{"field":"id","direction":"desc"}]}'); {"links": [], "collections": [{"id": "testcollection_2", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-170, -90, -160, -80]]}, "temporal": {"interval": [["2000-01-15 00:00:00+00", "2000-03-15 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_1", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-180, -90, -170, -80]]}, "temporal": {"interval": [["2000-01-08 00:00:00+00", "2000-03-08 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}], "numberMatched": 2, "numberReturned": 2} select collection_search('{"ids":["testcollection_1","testcollection_2"],"limit":10, "sortby":[{"field":"id","direction":"asc"}]}'); {"links": [], "collections": [{"id": "testcollection_1", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-180, -90, -170, -80]]}, "temporal": {"interval": [["2000-01-08 00:00:00+00", "2000-03-08 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_2", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-170, -90, -160, -80]]}, "temporal": {"interval": [["2000-01-15 00:00:00+00", "2000-03-15 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}], "numberMatched": 2, "numberReturned": 2} select collection_search('{"ids":["testcollection_1","testcollection_2","testcollection_3"],"limit":1, "sortby":[{"field":"id","direction":"desc"}]}'); {"links": [{"rel": "next", "body": {"offset": 1}, "href": "./collections", "type": "application/json", "merge": true, "method": "GET"}], "collections": [{"id": "testcollection_3", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-160, -90, -150, -80]]}, "temporal": {"interval": [["2000-01-22 00:00:00+00", "2000-03-22 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}], "numberMatched": 3, "numberReturned": 1} select collection_search('{"ids":["testcollection_1","testcollection_2"],"limit":1, "offset":10, "sortby":[{"field":"datetime","direction":"desc"}]}'); {"links": [{"rel": "prev", "body": {"offset": 9}, "href": "./collections", "type": "application/json", "merge": true, "method": "GET"}], "collections": [], "numberMatched": 2, "numberReturned": 0} select collection_search('{"filter":{"op":"eq", "args":[{"property":"title"},"My Test Collection."]},"limit":10, "sortby":[{"field":"datetime","direction":"desc"}]}'); {"links": [{"rel": "next", "body": {"offset": 10}, "href": "./collections", "type": "application/json", "merge": true, "method": "GET"}], "collections": [{"id": "testcollection_648", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[170, 80, 180, 90]]}, "temporal": {"interval": [["2012-06-02 00:00:00+00", "2012-08-02 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_647", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[160, 80, 170, 90]]}, "temporal": {"interval": [["2012-05-26 00:00:00+00", "2012-07-26 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_646", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[150, 80, 160, 90]]}, "temporal": {"interval": [["2012-05-19 00:00:00+00", "2012-07-19 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_645", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[140, 80, 150, 90]]}, "temporal": {"interval": [["2012-05-12 00:00:00+00", "2012-07-12 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_644", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[130, 80, 140, 90]]}, "temporal": {"interval": [["2012-05-05 00:00:00+00", "2012-07-05 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_643", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[120, 80, 130, 90]]}, "temporal": {"interval": [["2012-04-28 00:00:00+00", "2012-06-28 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_642", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[110, 80, 120, 90]]}, "temporal": {"interval": [["2012-04-21 00:00:00+00", "2012-06-21 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_641", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[100, 80, 110, 90]]}, "temporal": {"interval": [["2012-04-14 00:00:00+00", "2012-06-14 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_640", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[90, 80, 100, 90]]}, "temporal": {"interval": [["2012-04-07 00:00:00+00", "2012-06-07 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_639", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[80, 80, 90, 90]]}, "temporal": {"interval": [["2012-03-31 00:00:00+00", "2012-05-31 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}], "numberMatched": 648, "numberReturned": 10} select collection_search('{"datetime":["2012-01-01","2012-01-02"], "filter":{"op":"eq", "args":[{"property":"title"},"My Test Collection."]},"limit":10, "sortby":[{"field":"datetime","direction":"desc"}]}'); {"links": [], "collections": [{"id": "testcollection_626", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-50, 80, -40, 90]]}, "temporal": {"interval": [["2011-12-31 00:00:00+00", "2012-02-29 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_625", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-60, 80, -50, 90]]}, "temporal": {"interval": [["2011-12-24 00:00:00+00", "2012-02-24 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_624", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-70, 80, -60, 90]]}, "temporal": {"interval": [["2011-12-17 00:00:00+00", "2012-02-17 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_623", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-80, 80, -70, 90]]}, "temporal": {"interval": [["2011-12-10 00:00:00+00", "2012-02-10 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_622", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-90, 80, -80, 90]]}, "temporal": {"interval": [["2011-12-03 00:00:00+00", "2012-02-03 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_621", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-100, 80, -90, 90]]}, "temporal": {"interval": [["2011-11-26 00:00:00+00", "2012-01-26 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_620", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-110, 80, -100, 90]]}, "temporal": {"interval": [["2011-11-19 00:00:00+00", "2012-01-19 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_619", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-120, 80, -110, 90]]}, "temporal": {"interval": [["2011-11-12 00:00:00+00", "2012-01-12 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}, {"id": "testcollection_618", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-130, 80, -120, 90]]}, "temporal": {"interval": [["2011-11-05 00:00:00+00", "2012-01-05 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}], "numberMatched": 9, "numberReturned": 9} select collection_search('{"ids":["testcollection_1","testcollection_2"], "fields": {"include": ["title"]}}'); {"links": [], "collections": [{"id": "testcollection_1", "title": "My Test Collection."}, {"id": "testcollection_2", "title": "My Test Collection."}], "numberMatched": 2, "numberReturned": 2} ================================================ FILE: src/pgstac/tests/basic/cql2_searches.sql ================================================ SET ROLE pgstac_read; SET pgstac."default_filter_lang" TO 'cql2-json'; SELECT search('{"ids":["pgstac-test-item-0097"],"fields":{"include":["id"]}}'); SELECT search('{"ids":["pgstac-test-item-0097","pgstac-test-item-0003"],"fields":{"include":["id"]}}'); SELECT search('{"collections":["pgstac-test-collection"],"fields":{"include":["id"]}, "limit": 1}'); SELECT search('{"collections":["something"]}'); SELECT search('{"collections":["something"],"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"t_intersects", "args":[{"property":"datetime"},"2011-08-16T00:00:00Z/2011-08-17T00:00:00Z"]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"filter":{"op":"t_intersects", "args":[{"property":"datetime"},"2011-08-16T00:00:00Z/2011-08-17T00:00:00Z"]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"filter":{"op":"t_intersects", "args":[{"property":"datetime"},"2011-08-16T00:00:00Z/2011-08-17T00:00:00Z"]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"filter":{"op":"eq", "args":[{"property":"eo:cloud_cover"},36]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"filter":{"op":"lt", "args":[{"property":"eo:cloud_cover"},25]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"eo:cloud_cover","direction":"asc"}]}'); SELECT search('{"filter":{"op":"in","args":[{"property":"id"},["pgstac-test-item-0097"]]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"in","args":[{"property":"id"},["pgstac-test-item-0097","pgstac-test-item-0003"]]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"in","args":[{"property":"collection"},["pgstac-test-collection"]]},"fields":{"include":["id"]}, "limit": 1}'); SELECT search('{"filter":{"op":"in","args":[{"property":"collection"},["nonexistent"]]}}'); SELECT search('{"filter":{"op":"in","args":[{"property":"collection"},["nonexistent"]]}, "conf":{"context":"off"}}'); SELECT search('{"conf": {"nohydrate": true}, "limit": 2}'); SELECT search('{"filter":{"op":"in","args":[{"property":"naip:state"},["zz","xx"]]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"in","args":[{"property":"naip:year"},[2012,2013]]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"a_equals","args":[{"property":"proj:bbox"},[654842, 3423507, 661516, 3431125]]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"a_equals","args":[[654842, 3423507, 661516, 3431125],{"property":"proj:bbox"}]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"a_equals","args":[[654842, 3423507, 661516],{"property":"proj:bbox"}]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"a_overlaps","args":[{"property":"proj:bbox"},[654842, 3423507, 661516, 12345]]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"a_overlaps","args":[{"property":"proj:bbox"},[12345]]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"a_contains","args":[{"property":"proj:bbox"},[654842, 3423507, 661516]]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"a_contains","args":[{"property":"proj:bbox"},[654842, 3423507, 661516, 12345]]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"a_contained_by","args":[{"property":"proj:bbox"},[654842, 3423507, 661516]]},"fields":{"include":["id"]}}'); SELECT search('{"filter":{"op":"a_contained_by","args":[{"property":"proj:bbox"},[654842, 3423507, 661516, 3431125, 234324]]},"fields":{"include":["id"]}}'); -- Test Paging SELECT search('{"fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"token":"next:pgstac-test-item-0048", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"token":"next:pgstac-test-item-0016", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"token":"prev:pgstac-test-item-0030", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"token":"prev:pgstac-test-item-0054", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); -- Test paging without fields extension SELECT jsonb_path_query(search('{"fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'), '$.features[*].id'); SELECT jsonb_path_query(search('{"token":"next:pgstac-test-item-0048","sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'), '$.features[*].id'); SELECT jsonb_path_query(search('{"token":"next:pgstac-test-item-0016","sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'), '$.features[*].id'); SELECT jsonb_path_query(search('{"token":"prev:pgstac-test-item-0030","sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'), '$.features[*].id'); SELECT jsonb_path_query(search('{"token":"prev:pgstac-test-item-0054","sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'), '$.features[*].id'); SELECT search('{"collections": ["pgstac-test-collection"], "limit": 1}'); SELECT search('{"collections": ["pgstac-test-collection"], "limit": 1, "token": "next:pgstac-test-item-0001"}'); ================================================ FILE: src/pgstac/tests/basic/cql2_searches.sql.out ================================================ SET ROLE pgstac_read; SET SET pgstac."default_filter_lang" TO 'cql2-json'; SET SELECT search('{"ids":["pgstac-test-item-0097"],"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}], "numberMatched": 1, "numberReturned": 1} SELECT search('{"ids":["pgstac-test-item-0097","pgstac-test-item-0003"],"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}], "numberMatched": 2, "numberReturned": 2} SELECT search('{"collections":["pgstac-test-collection"],"fields":{"include":["id"]}, "limit": 1}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0003", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "numberMatched": 100, "numberReturned": 1} SELECT search('{"collections":["something"]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberMatched": 0, "numberReturned": 0} SELECT search('{"collections":["something"],"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberMatched": 0, "numberReturned": 0} SELECT search('{"filter":{"op":"t_intersects", "args":[{"property":"datetime"},"2011-08-16T00:00:00Z/2011-08-17T00:00:00Z"]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0016", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0007", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 59}}, {"id": "pgstac-test-item-0008", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 64}}, {"id": "pgstac-test-item-0009", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 61}}, {"id": "pgstac-test-item-0010", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 31}}, {"id": "pgstac-test-item-0011", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 41}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0014", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 17}}, {"id": "pgstac-test-item-0015", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 54}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 13}}], "numberMatched": 57, "numberReturned": 10} SELECT search('{"filter":{"op":"t_intersects", "args":[{"property":"datetime"},"2011-08-16T00:00:00Z/2011-08-17T00:00:00Z"]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0016", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0007", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 59}}, {"id": "pgstac-test-item-0008", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 64}}, {"id": "pgstac-test-item-0009", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 61}}, {"id": "pgstac-test-item-0010", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 31}}, {"id": "pgstac-test-item-0011", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 41}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0014", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 17}}, {"id": "pgstac-test-item-0015", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 54}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 13}}], "numberMatched": 57, "numberReturned": 10} SELECT search('{"filter":{"op":"t_intersects", "args":[{"property":"datetime"},"2011-08-16T00:00:00Z/2011-08-17T00:00:00Z"]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0016", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0007", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 59}}, {"id": "pgstac-test-item-0008", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 64}}, {"id": "pgstac-test-item-0009", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 61}}, {"id": "pgstac-test-item-0010", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 31}}, {"id": "pgstac-test-item-0011", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 41}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0014", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 17}}, {"id": "pgstac-test-item-0015", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 54}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 13}}], "numberMatched": 57, "numberReturned": 10} SELECT search('{"filter":{"op":"eq", "args":[{"property":"eo:cloud_cover"},36]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0087", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 36}}, {"id": "pgstac-test-item-0089", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 36}}], "numberMatched": 2, "numberReturned": 2} SELECT search('{"filter":{"op":"lt", "args":[{"property":"eo:cloud_cover"},25]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"eo:cloud_cover","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0012", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 1}}, {"id": "pgstac-test-item-0063", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0073", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0041", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0034", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0005", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0048", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}], "numberMatched": 31, "numberReturned": 10} SELECT search('{"filter":{"op":"in","args":[{"property":"id"},["pgstac-test-item-0097"]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}], "numberMatched": 1, "numberReturned": 1} SELECT search('{"filter":{"op":"in","args":[{"property":"id"},["pgstac-test-item-0097","pgstac-test-item-0003"]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}], "numberMatched": 2, "numberReturned": 2} SELECT search('{"filter":{"op":"in","args":[{"property":"collection"},["pgstac-test-collection"]]},"fields":{"include":["id"]}, "limit": 1}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0003", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "numberMatched": 100, "numberReturned": 1} SELECT search('{"filter":{"op":"in","args":[{"property":"collection"},["nonexistent"]]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberMatched": 0, "numberReturned": 0} SELECT search('{"filter":{"op":"in","args":[{"property":"collection"},["nonexistent"]]}, "conf":{"context":"off"}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberReturned": 0} SELECT search('{"conf": {"nohydrate": true}, "limit": 2}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0002", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]}, {"id": "pgstac-test-item-0002", "bbox": [-85.504167, 30.934008, -85.433293, 31.003486], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008505_nw_16_1_20110825.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.200.jpg"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.434414, 30.934008], [-85.433293, 31.002658], [-85.503096, 31.003486], [-85.504167, 30.934834], [-85.434414, 30.934008]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [642906, 3423339, 649572, 3430950], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7611, 6666], "eo:cloud_cover": 33, "proj:transform": [1, 0, 642906, 0, -1, 3430950, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]}], "numberMatched": 100, "numberReturned": 2} SELECT search('{"filter":{"op":"in","args":[{"property":"naip:state"},["zz","xx"]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0100", "collection": "pgstac-test-collection"}], "numberMatched": 2, "numberReturned": 2} SELECT search('{"filter":{"op":"in","args":[{"property":"naip:year"},[2012,2013]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0100", "collection": "pgstac-test-collection"}], "numberMatched": 2, "numberReturned": 2} SELECT search('{"filter":{"op":"a_equals","args":[{"property":"proj:bbox"},[654842, 3423507, 661516, 3431125]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "numberMatched": 1, "numberReturned": 1} SELECT search('{"filter":{"op":"a_equals","args":[[654842, 3423507, 661516, 3431125],{"property":"proj:bbox"}]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "numberMatched": 1, "numberReturned": 1} SELECT search('{"filter":{"op":"a_equals","args":[[654842, 3423507, 661516],{"property":"proj:bbox"}]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberMatched": 0, "numberReturned": 0} SELECT search('{"filter":{"op":"a_overlaps","args":[{"property":"proj:bbox"},[654842, 3423507, 661516, 12345]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "numberMatched": 1, "numberReturned": 1} SELECT search('{"filter":{"op":"a_overlaps","args":[{"property":"proj:bbox"},[12345]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberMatched": 0, "numberReturned": 0} SELECT search('{"filter":{"op":"a_contains","args":[{"property":"proj:bbox"},[654842, 3423507, 661516]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "numberMatched": 1, "numberReturned": 1} SELECT search('{"filter":{"op":"a_contains","args":[{"property":"proj:bbox"},[654842, 3423507, 661516, 12345]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberMatched": 0, "numberReturned": 0} SELECT search('{"filter":{"op":"a_contained_by","args":[{"property":"proj:bbox"},[654842, 3423507, 661516]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberMatched": 0, "numberReturned": 0} SELECT search('{"filter":{"op":"a_contained_by","args":[{"property":"proj:bbox"},[654842, 3423507, 661516, 3431125, 234324]]},"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "numberMatched": 1, "numberReturned": 1} -- Test Paging SELECT search('{"fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0048", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 1}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0063", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0005", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0034", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0041", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0073", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0048", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 4}}], "numberMatched": 100, "numberReturned": 10} SELECT search('{"token":"next:pgstac-test-item-0048", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0016", "type": "application/geo+json", "method": "GET"}, {"rel": "prev", "href": "./search?token=prev:pgstac-test-collection:pgstac-test-item-0054", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0054", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 5}}, {"id": "pgstac-test-item-0032", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 7}}, {"id": "pgstac-test-item-0057", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 7}}, {"id": "pgstac-test-item-0075", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 8}}, {"id": "pgstac-test-item-0049", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 9}}, {"id": "pgstac-test-item-0040", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 10}}, {"id": "pgstac-test-item-0036", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 12}}, {"id": "pgstac-test-item-0039", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 12}}, {"id": "pgstac-test-item-0052", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 12}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 13}}], "numberMatched": 100, "numberReturned": 10} SELECT search('{"token":"next:pgstac-test-item-0016", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0053", "type": "application/geo+json", "method": "GET"}, {"rel": "prev", "href": "./search?token=prev:pgstac-test-collection:pgstac-test-item-0030", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0030", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 16}}, {"id": "pgstac-test-item-0031", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 16}}, {"id": "pgstac-test-item-0046", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 16}}, {"id": "pgstac-test-item-0014", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 17}}, {"id": "pgstac-test-item-0078", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 21}}, {"id": "pgstac-test-item-0051", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 22}}, {"id": "pgstac-test-item-0068", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 22}}, {"id": "pgstac-test-item-0004", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 23}}, {"id": "pgstac-test-item-0088", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 23}}, {"id": "pgstac-test-item-0053", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 24}}], "numberMatched": 100, "numberReturned": 10} SELECT search('{"token":"prev:pgstac-test-item-0030", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0016", "type": "application/geo+json", "method": "GET"}, {"rel": "prev", "href": "./search?token=prev:pgstac-test-collection:pgstac-test-item-0054", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0054", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 5}}, {"id": "pgstac-test-item-0032", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 7}}, {"id": "pgstac-test-item-0057", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 7}}, {"id": "pgstac-test-item-0075", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 8}}, {"id": "pgstac-test-item-0049", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 9}}, {"id": "pgstac-test-item-0040", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 10}}, {"id": "pgstac-test-item-0036", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 12}}, {"id": "pgstac-test-item-0039", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 12}}, {"id": "pgstac-test-item-0052", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 12}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 13}}], "numberMatched": 100, "numberReturned": 10} SELECT search('{"token":"prev:pgstac-test-item-0054", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0048", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 1}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0063", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0005", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0034", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0041", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0073", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0048", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 4}}], "numberMatched": 100, "numberReturned": 10} -- Test paging without fields extension SELECT jsonb_path_query(search('{"fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'), '$.features[*].id'); "pgstac-test-item-0097" "pgstac-test-item-0013" "pgstac-test-item-0063" "pgstac-test-item-0005" "pgstac-test-item-0034" "pgstac-test-item-0041" "pgstac-test-item-0073" "pgstac-test-item-0085" "pgstac-test-item-0012" "pgstac-test-item-0048" SELECT jsonb_path_query(search('{"token":"next:pgstac-test-item-0048","sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'), '$.features[*].id'); "pgstac-test-item-0054" "pgstac-test-item-0032" "pgstac-test-item-0057" "pgstac-test-item-0075" "pgstac-test-item-0049" "pgstac-test-item-0040" "pgstac-test-item-0036" "pgstac-test-item-0039" "pgstac-test-item-0052" "pgstac-test-item-0016" SELECT jsonb_path_query(search('{"token":"next:pgstac-test-item-0016","sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'), '$.features[*].id'); "pgstac-test-item-0030" "pgstac-test-item-0031" "pgstac-test-item-0046" "pgstac-test-item-0014" "pgstac-test-item-0078" "pgstac-test-item-0051" "pgstac-test-item-0068" "pgstac-test-item-0004" "pgstac-test-item-0088" "pgstac-test-item-0053" SELECT jsonb_path_query(search('{"token":"prev:pgstac-test-item-0030","sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'), '$.features[*].id'); "pgstac-test-item-0054" "pgstac-test-item-0032" "pgstac-test-item-0057" "pgstac-test-item-0075" "pgstac-test-item-0049" "pgstac-test-item-0040" "pgstac-test-item-0036" "pgstac-test-item-0039" "pgstac-test-item-0052" "pgstac-test-item-0016" SELECT jsonb_path_query(search('{"token":"prev:pgstac-test-item-0054","sortby":[{"field":"properties.eo:cloud_cover","direction":"asc"},{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'), '$.features[*].id'); "pgstac-test-item-0097" "pgstac-test-item-0013" "pgstac-test-item-0063" "pgstac-test-item-0005" "pgstac-test-item-0034" "pgstac-test-item-0041" "pgstac-test-item-0073" "pgstac-test-item-0085" "pgstac-test-item-0012" "pgstac-test-item-0048" SELECT search('{"collections": ["pgstac-test-collection"], "limit": 1}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0003", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}], "numberMatched": 100, "numberReturned": 1} SELECT search('{"collections": ["pgstac-test-collection"], "limit": 1, "token": "next:pgstac-test-item-0001"}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0006", "type": "application/geo+json", "method": "GET"}, {"rel": "prev", "href": "./search?token=prev:pgstac-test-collection:pgstac-test-item-0006", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0006", "bbox": [-87.253322, 30.934677, -87.184223, 31.00282], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_nw_16_1_20110824.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008707_nw_16_1_20110824.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_nw_16_1_20110824.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.184223, 30.934794], [-87.184354, 31.00282], [-87.253322, 31.002703], [-87.253142, 30.934677], [-87.184223, 30.934794]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [475817, 3422390, 482401, 3429929], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7539, 6584], "eo:cloud_cover": 100, "proj:transform": [1, 0, 475817, 0, -1, 3429929, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}], "numberMatched": 100, "numberReturned": 1} ================================================ FILE: src/pgstac/tests/basic/cql_searches.sql ================================================ SET pgstac."default_filter_lang" TO 'cql-json'; SELECT search('{"fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); -- Test Paging SELECT search('{"fields":{"include":["id"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"token":"next:pgstac-test-item-0010", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"token":"next:pgstac-test-item-0020", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"token":"prev:pgstac-test-item-0021", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"token":"next:pgstac-test-item-0011", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); -- SELECT search('{"datetime":"2011-08-16T00:00:00Z/2011-08-17T00:00:00Z", "fields":{"include":["id","properties.datetime"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"datetime":["2011-08-16T00:00:00Z","2011-08-17T00:00:00Z"], "fields":{"include":["id","properties.datetime"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"filter":{"anyinteracts":[{"property":"datetime"},["2011-08-16T00:00:00Z","2011-08-17T00:00:00Z"]]}, "fields":{"include":["id","properties.datetime"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"filter":{"eq":[{"property":"eo:cloud_cover"},36]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); SELECT search('{"filter":{"lt":[{"property":"eo:cloud_cover"},25]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"eo:cloud_cover","direction":"asc"}]}'); SELECT search('{"ids":["pgstac-test-item-0097"],"fields":{"include":["id"]}}'); SELECT search('{"ids":["pgstac-test-item-0097","pgstac-test-item-0003"],"fields":{"include":["id"]}}'); SELECT search('{"ids":["bogusid"],"fields":{"include":["id"]}}'); SELECT search('{"collections":["pgstac-test-collection"],"fields":{"include":["id"]}, "limit": 1}'); SELECT search('{"collections":["something"]}'); SELECT search('{"collections":["something"],"fields":{"include":["id"]}}'); SELECT usecount IS NOT NULL and usecount > 0 AND lastused IS NOT NULL AND lastused < clock_timestamp() FROM search_query(jsonb_build_object('collections',ARRAY[random()::text])); SELECT hash, search, _where, orderby, metadata from search_query('{"collections":["pgstac-test-collection"]}'::jsonb, _metadata=>'{"meta":"value"}'::jsonb); SELECT hash, search, _where, orderby, metadata from search_query('{"collections":["pgstac-test-collection"]}'::jsonb, _metadata=>'{"meta":"value"}'::jsonb); SELECT usecount IS NOT NULL and usecount > 0 AND lastused IS NOT NULL AND lastused < clock_timestamp() FROM search_query('{"collections":["pgstac-test-collection"]}'); ================================================ FILE: src/pgstac/tests/basic/cql_searches.sql.out ================================================ SET pgstac."default_filter_lang" TO 'cql-json'; SET SELECT search('{"fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0010", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0001", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-25T00:00:00Z", "eo:cloud_cover": 89}}, {"id": "pgstac-test-item-0002", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-25T00:00:00Z", "eo:cloud_cover": 33}}, {"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-25T00:00:00Z", "eo:cloud_cover": 28}}, {"id": "pgstac-test-item-0004", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 23}}, {"id": "pgstac-test-item-0005", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0006", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 100}}, {"id": "pgstac-test-item-0007", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 59}}, {"id": "pgstac-test-item-0008", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 64}}, {"id": "pgstac-test-item-0009", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 61}}, {"id": "pgstac-test-item-0010", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 31}}], "numberMatched": 100, "numberReturned": 10} -- Test Paging SELECT search('{"fields":{"include":["id"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0010", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0001", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0002", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0004", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0005", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0006", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0007", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0008", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0009", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0010", "collection": "pgstac-test-collection"}], "numberMatched": 100, "numberReturned": 10} SELECT search('{"token":"next:pgstac-test-item-0010", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0020", "type": "application/geo+json", "method": "GET"}, {"rel": "prev", "href": "./search?token=prev:pgstac-test-collection:pgstac-test-item-0011", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0011", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 41}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0014", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 17}}, {"id": "pgstac-test-item-0015", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 54}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 13}}, {"id": "pgstac-test-item-0017", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 59}}, {"id": "pgstac-test-item-0018", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 29}}, {"id": "pgstac-test-item-0019", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 52}}, {"id": "pgstac-test-item-0020", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 39}}], "numberMatched": 100, "numberReturned": 10} SELECT search('{"token":"next:pgstac-test-item-0020", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0030", "type": "application/geo+json", "method": "GET"}, {"rel": "prev", "href": "./search?token=prev:pgstac-test-collection:pgstac-test-item-0021", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0021", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 29}}, {"id": "pgstac-test-item-0022", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 84}}, {"id": "pgstac-test-item-0023", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 56}}, {"id": "pgstac-test-item-0024", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 58}}, {"id": "pgstac-test-item-0025", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 65}}, {"id": "pgstac-test-item-0026", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 52}}, {"id": "pgstac-test-item-0027", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 43}}, {"id": "pgstac-test-item-0028", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 46}}, {"id": "pgstac-test-item-0029", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 42}}, {"id": "pgstac-test-item-0030", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 16}}], "numberMatched": 100, "numberReturned": 10} SELECT search('{"token":"prev:pgstac-test-item-0021", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0020", "type": "application/geo+json", "method": "GET"}, {"rel": "prev", "href": "./search?token=prev:pgstac-test-collection:pgstac-test-item-0011", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0011", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 41}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0014", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 17}}, {"id": "pgstac-test-item-0015", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 54}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 13}}, {"id": "pgstac-test-item-0017", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 59}}, {"id": "pgstac-test-item-0018", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 29}}, {"id": "pgstac-test-item-0019", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 52}}, {"id": "pgstac-test-item-0020", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 39}}], "numberMatched": 100, "numberReturned": 10} SELECT search('{"token":"next:pgstac-test-item-0011", "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0021", "type": "application/geo+json", "method": "GET"}, {"rel": "prev", "href": "./search?token=prev:pgstac-test-collection:pgstac-test-item-0012", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0014", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 17}}, {"id": "pgstac-test-item-0015", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 54}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 13}}, {"id": "pgstac-test-item-0017", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 59}}, {"id": "pgstac-test-item-0018", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 29}}, {"id": "pgstac-test-item-0019", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 52}}, {"id": "pgstac-test-item-0020", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 39}}, {"id": "pgstac-test-item-0021", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 29}}], "numberMatched": 100, "numberReturned": 10} -- SELECT search('{"datetime":"2011-08-16T00:00:00Z/2011-08-17T00:00:00Z", "fields":{"include":["id","properties.datetime"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0016", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0007", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0008", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0009", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0010", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0011", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0014", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z"}}, {"id": "pgstac-test-item-0015", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z"}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z"}}], "numberMatched": 57, "numberReturned": 10} SELECT search('{"datetime":["2011-08-16T00:00:00Z","2011-08-17T00:00:00Z"], "fields":{"include":["id","properties.datetime"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0016", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0007", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0008", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0009", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0010", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0011", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0014", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z"}}, {"id": "pgstac-test-item-0015", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z"}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z"}}], "numberMatched": 57, "numberReturned": 10} SELECT search('{"filter":{"anyinteracts":[{"property":"datetime"},["2011-08-16T00:00:00Z","2011-08-17T00:00:00Z"]]}, "fields":{"include":["id","properties.datetime"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0016", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0007", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0008", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0009", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0010", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0011", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z"}}, {"id": "pgstac-test-item-0014", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z"}}, {"id": "pgstac-test-item-0015", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z"}}, {"id": "pgstac-test-item-0016", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z"}}], "numberMatched": 57, "numberReturned": 10} SELECT search('{"filter":{"eq":[{"property":"eo:cloud_cover"},36]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"datetime","direction":"desc"},{"field":"id","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0087", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 36}}, {"id": "pgstac-test-item-0089", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 36}}], "numberMatched": 2, "numberReturned": 2} SELECT search('{"filter":{"lt":[{"property":"eo:cloud_cover"},25]}, "fields":{"include":["id","properties.datetime","properties.eo:cloud_cover"]},"sortby":[{"field":"eo:cloud_cover","direction":"asc"}]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0012", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-07-31T00:00:00Z", "eo:cloud_cover": 1}}, {"id": "pgstac-test-item-0063", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0013", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 2}}, {"id": "pgstac-test-item-0085", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-01T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0073", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-15T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0041", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0034", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0005", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-24T00:00:00Z", "eo:cloud_cover": 3}}, {"id": "pgstac-test-item-0048", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-16T00:00:00Z", "eo:cloud_cover": 4}}, {"id": "pgstac-test-item-0012", "collection": "pgstac-test-collection", "properties": {"datetime": "2011-08-17T00:00:00Z", "eo:cloud_cover": 4}}], "numberMatched": 31, "numberReturned": 10} SELECT search('{"ids":["pgstac-test-item-0097"],"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}], "numberMatched": 1, "numberReturned": 1} SELECT search('{"ids":["pgstac-test-item-0097","pgstac-test-item-0003"],"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}], "numberMatched": 2, "numberReturned": 2} SELECT search('{"ids":["bogusid"],"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberMatched": 0, "numberReturned": 0} SELECT search('{"collections":["pgstac-test-collection"],"fields":{"include":["id"]}, "limit": 1}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}, {"rel": "next", "href": "./search?token=next:pgstac-test-collection:pgstac-test-item-0003", "type": "application/geo+json", "method": "GET"}], "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}], "numberMatched": 100, "numberReturned": 1} SELECT search('{"collections":["something"]}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberMatched": 0, "numberReturned": 0} SELECT search('{"collections":["something"],"fields":{"include":["id"]}}'); {"type": "FeatureCollection", "links": [{"rel": "root", "href": ".", "type": "application/json"}, {"rel": "self", "href": "./search", "type": "application/json"}], "features": [], "numberMatched": 0, "numberReturned": 0} SELECT usecount IS NOT NULL and usecount > 0 AND lastused IS NOT NULL AND lastused < clock_timestamp() FROM search_query(jsonb_build_object('collections',ARRAY[random()::text])); t SELECT hash, search, _where, orderby, metadata from search_query('{"collections":["pgstac-test-collection"]}'::jsonb, _metadata=>'{"meta":"value"}'::jsonb); 06efe6c09f0d61fd212e882325041a73 | {"collections": ["pgstac-test-collection"]} | collection = ANY ('{pgstac-test-collection}') | datetime DESC, id DESC | {"meta": "value"} SELECT hash, search, _where, orderby, metadata from search_query('{"collections":["pgstac-test-collection"]}'::jsonb, _metadata=>'{"meta":"value"}'::jsonb); 06efe6c09f0d61fd212e882325041a73 | {"collections": ["pgstac-test-collection"]} | collection = ANY ('{pgstac-test-collection}') | datetime DESC, id DESC | {"meta": "value"} SELECT usecount IS NOT NULL and usecount > 0 AND lastused IS NOT NULL AND lastused < clock_timestamp() FROM search_query('{"collections":["pgstac-test-collection"]}'); t ================================================ FILE: src/pgstac/tests/basic/crud_functions.sql ================================================ -- run tests as pgstac_ingest SET ROLE pgstac_ingest; SET pgstac.use_queue=FALSE; SELECT get_setting_bool('use_queue'); SET pgstac.update_collection_extent=TRUE; SELECT get_setting_bool('update_collection_extent'); --create base data to use with tests CREATE TEMP TABLE test_items AS SELECT jsonb_build_object( 'id', concat('pgstactest-crudtest-', (row_number() over ())::text), 'collection', 'pgstactest-crudtest', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object( 'datetime', g::text) ) as content FROM generate_series('2020-01-01'::timestamptz, '2022-01-01'::timestamptz, '1 month'::interval) g; --test collection partioned by year INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-crudtest"}', 'year'); -- Create an item SELECT create_item((SELECT content FROM test_items LIMIT 1)); SELECT * FROM items WHERE collection='pgstactest-crudtest'; -- Check to see if extent got updated SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; -- Update item with new datetime that is in a different partition SELECT update_item((SELECT content || '{"properties":{"datetime":"2023-01-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); SELECT * FROM items WHERE collection='pgstactest-crudtest'; -- Check to see if extent got updated SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; -- Update item with new datetime that is in a different partition SELECT upsert_item((SELECT content || '{"properties":{"datetime":"2023-02-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); SELECT * FROM items WHERE collection='pgstactest-crudtest'; -- Delete an item SELECT delete_item('pgstactest-crudtest-1', 'pgstactest-crudtest'); SELECT * FROM items WHERE collection='pgstactest-crudtest'; WITH c AS (SELECT content FROM test_items LIMIT 2), aggregated AS (SELECT jsonb_agg(content) as items FROM c) SELECT create_items(items) FROM aggregated; SELECT * FROM items WHERE collection='pgstactest-crudtest'; DELETE FROM items WHERE collection='pgstactest-crudtest'; -- upsert items that do not exist yet WITH c AS (SELECT content FROM test_items LIMIT 2), aggregated AS (SELECT jsonb_agg(content) as items FROM c) SELECT upsert_items(items) FROM aggregated; SELECT * FROM items WHERE collection='pgstactest-crudtest'; -- upsert items that already exist and are to be modified WITH c AS (SELECT content || '{"properties":{"datetime":"2023-02-01 00:00:00Z"}}'::jsonb as content FROM test_items LIMIT 2), aggregated AS (SELECT jsonb_agg(content) as items FROM c) SELECT upsert_items(items) FROM aggregated; SELECT * FROM items WHERE collection='pgstactest-crudtest'; -- turn off update_collection_extent then add an item and verify that the extent did not get updated automatically SET pgstac.update_collection_extent=FALSE; SELECT get_setting_bool('update_collection_extent'); SELECT update_item((SELECT content || '{"properties":{"datetime":"2024-01-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; -- check formatting of temporal extent SELECT collection_temporal_extent('pgstactest-crudtest'); ================================================ FILE: src/pgstac/tests/basic/crud_functions.sql.out ================================================ -- run tests as pgstac_ingest SET ROLE pgstac_ingest; SET SET pgstac.use_queue=FALSE; SET SELECT get_setting_bool('use_queue'); f SET pgstac.update_collection_extent=TRUE; SET SELECT get_setting_bool('update_collection_extent'); t --create base data to use with tests CREATE TEMP TABLE test_items AS SELECT jsonb_build_object( 'id', concat('pgstactest-crudtest-', (row_number() over ())::text), 'collection', 'pgstactest-crudtest', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object( 'datetime', g::text) ) as content FROM generate_series('2020-01-01'::timestamptz, '2022-01-01'::timestamptz, '1 month'::interval) g; SELECT 25 --test collection partioned by year INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-crudtest"}', 'year'); INSERT 0 1 -- Create an item SELECT create_item((SELECT content FROM test_items LIMIT 1)); SELECT * FROM items WHERE collection='pgstactest-crudtest'; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2020-01-01 00:00:00+00 | 2020-01-01 00:00:00+00 | {"properties": {"datetime": "2020-01-01 00:00:00+00"}} | -- Check to see if extent got updated SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2020-01-01T00:00:00+00:00"]]}} -- Update item with new datetime that is in a different partition SELECT update_item((SELECT content || '{"properties":{"datetime":"2023-01-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); SELECT * FROM items WHERE collection='pgstactest-crudtest'; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2023-01-01 00:00:00+00 | 2023-01-01 00:00:00+00 | {"properties": {"datetime": "2023-01-01 00:00:00Z"}} | -- Check to see if extent got updated SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2023-01-01T00:00:00+00:00", "2023-01-01T00:00:00+00:00"]]}} -- Update item with new datetime that is in a different partition SELECT upsert_item((SELECT content || '{"properties":{"datetime":"2023-02-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); SELECT * FROM items WHERE collection='pgstactest-crudtest'; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2023-02-01 00:00:00+00 | 2023-02-01 00:00:00+00 | {"properties": {"datetime": "2023-02-01 00:00:00Z"}} | -- Delete an item SELECT delete_item('pgstactest-crudtest-1', 'pgstactest-crudtest'); SELECT * FROM items WHERE collection='pgstactest-crudtest'; WITH c AS (SELECT content FROM test_items LIMIT 2), aggregated AS (SELECT jsonb_agg(content) as items FROM c) SELECT create_items(items) FROM aggregated; SELECT * FROM items WHERE collection='pgstactest-crudtest'; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2020-01-01 00:00:00+00 | 2020-01-01 00:00:00+00 | {"properties": {"datetime": "2020-01-01 00:00:00+00"}} | pgstactest-crudtest-2 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2020-02-01 00:00:00+00 | 2020-02-01 00:00:00+00 | {"properties": {"datetime": "2020-02-01 00:00:00+00"}} | DELETE FROM items WHERE collection='pgstactest-crudtest'; DELETE 2 -- upsert items that do not exist yet WITH c AS (SELECT content FROM test_items LIMIT 2), aggregated AS (SELECT jsonb_agg(content) as items FROM c) SELECT upsert_items(items) FROM aggregated; SELECT * FROM items WHERE collection='pgstactest-crudtest'; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2020-01-01 00:00:00+00 | 2020-01-01 00:00:00+00 | {"properties": {"datetime": "2020-01-01 00:00:00+00"}} | pgstactest-crudtest-2 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2020-02-01 00:00:00+00 | 2020-02-01 00:00:00+00 | {"properties": {"datetime": "2020-02-01 00:00:00+00"}} | -- upsert items that already exist and are to be modified WITH c AS (SELECT content || '{"properties":{"datetime":"2023-02-01 00:00:00Z"}}'::jsonb as content FROM test_items LIMIT 2), aggregated AS (SELECT jsonb_agg(content) as items FROM c) SELECT upsert_items(items) FROM aggregated; SELECT * FROM items WHERE collection='pgstactest-crudtest'; pgstactest-crudtest-1 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2023-02-01 00:00:00+00 | 2023-02-01 00:00:00+00 | {"properties": {"datetime": "2023-02-01 00:00:00Z"}} | pgstactest-crudtest-2 | 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 | pgstactest-crudtest | 2023-02-01 00:00:00+00 | 2023-02-01 00:00:00+00 | {"properties": {"datetime": "2023-02-01 00:00:00Z"}} | -- turn off update_collection_extent then add an item and verify that the extent did not get updated automatically SET pgstac.update_collection_extent=FALSE; SET SELECT get_setting_bool('update_collection_extent'); f SELECT update_item((SELECT content || '{"properties":{"datetime":"2024-01-01 00:00:00Z"}}'::jsonb FROM test_items LIMIT 1)); SELECT content->'extent' FROM collections WHERE id='pgstactest-crudtest'; {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2023-02-01T00:00:00+00:00", "2023-02-01T00:00:00+00:00"]]}} -- check formatting of temporal extent SELECT collection_temporal_extent('pgstactest-crudtest'); [["2023-02-01T00:00:00+00:00", "2024-01-01T00:00:00+00:00"]] ================================================ FILE: src/pgstac/tests/basic/free_text.sql ================================================ SET pgstac.context TO 'on'; SET pgstac."default_filter_lang" TO 'cql2-json'; CREATE TEMP TABLE temp_collections ( id SERIAL PRIMARY KEY, title TEXT, description TEXT, keywords TEXT, minx NUMERIC, miny NUMERIC, maxx NUMERIC, maxy NUMERIC, sdt TIMESTAMPTZ, edt TIMESTAMPTZ ); INSERT INTO temp_collections ( title, description, keywords, minx, miny, maxx, maxy, sdt, edt ) VALUES -- no keywords ( 'Stranger Things', 'Some teenagers drop out of school to fight scary monsters', null, -180, -90, 180, 90, '2016-01-01T00:00:00Z', '2025-12-31T23:59:59Z' ), ( 'The Bear', 'Another story about why you should not start a restaurant', 'restaurant, funny, sad, great', -180, -90, 180, 90, '2022-01-01T00:00:00Z', '2025-12-31T23:59:59Z' ), ( 'Godzilla', 'A large lizard takes its revenge', 'scary, lizard, monster', -180, -90, 180, 90, '1954-01-01T00:00:00Z', null ), ( 'Chefs Table', 'Another great story that make you wonder if you should go to a restaurant', 'restaurant, food, michelin', -180, -90, 180, 90, '2019-01-01T00:00:00Z', '2025-12-31T23:59:59Z' ), -- no title ( null, 'A humoristic portrayal of office life', 'Scranton, paper', -180, -90, 180, 90, '2005-01-01T00:00:00Z', '2013-12-31T23:59:59Z' ); SELECT create_collection(jsonb_build_object( 'id', format('testcollection_%s', id), 'type', 'Collection', 'title', title, 'description', description, 'extent', jsonb_build_object( 'spatial', jsonb_build_array(jsonb_build_array(minx, miny, maxx, maxy)), 'temporal', jsonb_build_array(jsonb_build_array(sdt, edt)) ), 'stac_extensions', jsonb_build_array(), 'keywords', string_to_array(keywords, ', ') )) FROM temp_collections; select collection_search('{"q": "monsters"}'); select collection_search('{"q": "lizard"}'); select collection_search('{"q": "scary OR funny"}'); select collection_search('{"q": "(scary AND revenge) OR (funny AND sad)"}'); select collection_search('{"q": "\"great story\""}'); select collection_search('{"q": "monster -school"}'); select collection_search('{"q": "+restaurant -sad"}'); select collection_search('{"q": "+restaurant"}'); select collection_search('{"q": "bear or stranger"}'); select collection_search('{"q": "bear OR stranger"}'); select collection_search('{"q": "bear, stranger"}'); select collection_search('{"q": "bear AND stranger"}'); select collection_search('{"q": "bear and stranger"}'); select collection_search('{"q": "\"bear or stranger\""}'); select collection_search('{"q": "office"}'); select collection_search('{"q": ["bear", "stranger"]}'); select collection_search('{"q": "large lizard"}'); select collection_search('{"q": "teenagers fight monsters"}'); select collection_search('{"q": "scary monsters"}'); ================================================ FILE: src/pgstac/tests/basic/free_text.sql.out ================================================ SET pgstac.context TO 'on'; SET SET pgstac."default_filter_lang" TO 'cql2-json'; SET CREATE TEMP TABLE temp_collections ( id SERIAL PRIMARY KEY, title TEXT, description TEXT, keywords TEXT, minx NUMERIC, miny NUMERIC, maxx NUMERIC, maxy NUMERIC, sdt TIMESTAMPTZ, edt TIMESTAMPTZ ); CREATE TABLE INSERT INTO temp_collections ( title, description, keywords, minx, miny, maxx, maxy, sdt, edt ) VALUES -- no keywords ( 'Stranger Things', 'Some teenagers drop out of school to fight scary monsters', null, -180, -90, 180, 90, '2016-01-01T00:00:00Z', '2025-12-31T23:59:59Z' ), ( 'The Bear', 'Another story about why you should not start a restaurant', 'restaurant, funny, sad, great', -180, -90, 180, 90, '2022-01-01T00:00:00Z', '2025-12-31T23:59:59Z' ), ( 'Godzilla', 'A large lizard takes its revenge', 'scary, lizard, monster', -180, -90, 180, 90, '1954-01-01T00:00:00Z', null ), ( 'Chefs Table', 'Another great story that make you wonder if you should go to a restaurant', 'restaurant, food, michelin', -180, -90, 180, 90, '2019-01-01T00:00:00Z', '2025-12-31T23:59:59Z' ), -- no title ( null, 'A humoristic portrayal of office life', 'Scranton, paper', -180, -90, 180, 90, '2005-01-01T00:00:00Z', '2013-12-31T23:59:59Z' ); INSERT 0 5 SELECT create_collection(jsonb_build_object( 'id', format('testcollection_%s', id), 'type', 'Collection', 'title', title, 'description', description, 'extent', jsonb_build_object( 'spatial', jsonb_build_array(jsonb_build_array(minx, miny, maxx, maxy)), 'temporal', jsonb_build_array(jsonb_build_array(sdt, edt)) ), 'stac_extensions', jsonb_build_array(), 'keywords', string_to_array(keywords, ', ') )) FROM temp_collections; select collection_search('{"q": "monsters"}'); {"links": [], "collections": [{"id": "testcollection_1", "type": "Collection", "title": "Stranger Things", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2016-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": null, "description": "Some teenagers drop out of school to fight scary monsters", "stac_extensions": []}, {"id": "testcollection_3", "type": "Collection", "title": "Godzilla", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["1954-01-01T00:00:00+00:00", null]]}, "keywords": ["scary", "lizard", "monster"], "description": "A large lizard takes its revenge", "stac_extensions": []}], "numberMatched": 2, "numberReturned": 2} select collection_search('{"q": "lizard"}'); {"links": [], "collections": [{"id": "testcollection_3", "type": "Collection", "title": "Godzilla", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["1954-01-01T00:00:00+00:00", null]]}, "keywords": ["scary", "lizard", "monster"], "description": "A large lizard takes its revenge", "stac_extensions": []}], "numberMatched": 1, "numberReturned": 1} select collection_search('{"q": "scary OR funny"}'); {"links": [], "collections": [{"id": "testcollection_1", "type": "Collection", "title": "Stranger Things", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2016-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": null, "description": "Some teenagers drop out of school to fight scary monsters", "stac_extensions": []}, {"id": "testcollection_2", "type": "Collection", "title": "The Bear", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2022-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": ["restaurant", "funny", "sad", "great"], "description": "Another story about why you should not start a restaurant", "stac_extensions": []}, {"id": "testcollection_3", "type": "Collection", "title": "Godzilla", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["1954-01-01T00:00:00+00:00", null]]}, "keywords": ["scary", "lizard", "monster"], "description": "A large lizard takes its revenge", "stac_extensions": []}], "numberMatched": 3, "numberReturned": 3} select collection_search('{"q": "(scary AND revenge) OR (funny AND sad)"}'); {"links": [], "collections": [{"id": "testcollection_2", "type": "Collection", "title": "The Bear", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2022-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": ["restaurant", "funny", "sad", "great"], "description": "Another story about why you should not start a restaurant", "stac_extensions": []}, {"id": "testcollection_3", "type": "Collection", "title": "Godzilla", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["1954-01-01T00:00:00+00:00", null]]}, "keywords": ["scary", "lizard", "monster"], "description": "A large lizard takes its revenge", "stac_extensions": []}], "numberMatched": 2, "numberReturned": 2} select collection_search('{"q": "\"great story\""}'); {"links": [], "collections": [{"id": "testcollection_4", "type": "Collection", "title": "Chefs Table", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2019-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": ["restaurant", "food", "michelin"], "description": "Another great story that make you wonder if you should go to a restaurant", "stac_extensions": []}], "numberMatched": 1, "numberReturned": 1} select collection_search('{"q": "monster -school"}'); {"links": [], "collections": [{"id": "testcollection_3", "type": "Collection", "title": "Godzilla", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["1954-01-01T00:00:00+00:00", null]]}, "keywords": ["scary", "lizard", "monster"], "description": "A large lizard takes its revenge", "stac_extensions": []}], "numberMatched": 1, "numberReturned": 1} select collection_search('{"q": "+restaurant -sad"}'); {"links": [], "collections": [{"id": "testcollection_4", "type": "Collection", "title": "Chefs Table", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2019-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": ["restaurant", "food", "michelin"], "description": "Another great story that make you wonder if you should go to a restaurant", "stac_extensions": []}], "numberMatched": 1, "numberReturned": 1} select collection_search('{"q": "+restaurant"}'); {"links": [], "collections": [{"id": "testcollection_2", "type": "Collection", "title": "The Bear", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2022-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": ["restaurant", "funny", "sad", "great"], "description": "Another story about why you should not start a restaurant", "stac_extensions": []}, {"id": "testcollection_4", "type": "Collection", "title": "Chefs Table", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2019-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": ["restaurant", "food", "michelin"], "description": "Another great story that make you wonder if you should go to a restaurant", "stac_extensions": []}], "numberMatched": 2, "numberReturned": 2} select collection_search('{"q": "bear or stranger"}'); {"links": [], "collections": [{"id": "testcollection_1", "type": "Collection", "title": "Stranger Things", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2016-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": null, "description": "Some teenagers drop out of school to fight scary monsters", "stac_extensions": []}, {"id": "testcollection_2", "type": "Collection", "title": "The Bear", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2022-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": ["restaurant", "funny", "sad", "great"], "description": "Another story about why you should not start a restaurant", "stac_extensions": []}], "numberMatched": 2, "numberReturned": 2} select collection_search('{"q": "bear OR stranger"}'); {"links": [], "collections": [{"id": "testcollection_1", "type": "Collection", "title": "Stranger Things", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2016-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": null, "description": "Some teenagers drop out of school to fight scary monsters", "stac_extensions": []}, {"id": "testcollection_2", "type": "Collection", "title": "The Bear", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2022-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": ["restaurant", "funny", "sad", "great"], "description": "Another story about why you should not start a restaurant", "stac_extensions": []}], "numberMatched": 2, "numberReturned": 2} select collection_search('{"q": "bear, stranger"}'); {"links": [], "collections": [{"id": "testcollection_1", "type": "Collection", "title": "Stranger Things", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2016-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": null, "description": "Some teenagers drop out of school to fight scary monsters", "stac_extensions": []}, {"id": "testcollection_2", "type": "Collection", "title": "The Bear", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2022-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": ["restaurant", "funny", "sad", "great"], "description": "Another story about why you should not start a restaurant", "stac_extensions": []}], "numberMatched": 2, "numberReturned": 2} select collection_search('{"q": "bear AND stranger"}'); {"links": [], "collections": [], "numberMatched": 0, "numberReturned": 0} select collection_search('{"q": "bear and stranger"}'); {"links": [], "collections": [], "numberMatched": 0, "numberReturned": 0} select collection_search('{"q": "\"bear or stranger\""}'); {"links": [], "collections": [], "numberMatched": 0, "numberReturned": 0} select collection_search('{"q": "office"}'); {"links": [], "collections": [{"id": "testcollection_5", "type": "Collection", "title": null, "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2005-01-01T00:00:00+00:00", "2013-12-31T23:59:59+00:00"]]}, "keywords": ["Scranton", "paper"], "description": "A humoristic portrayal of office life", "stac_extensions": []}], "numberMatched": 1, "numberReturned": 1} select collection_search('{"q": ["bear", "stranger"]}'); {"links": [], "collections": [{"id": "testcollection_1", "type": "Collection", "title": "Stranger Things", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2016-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": null, "description": "Some teenagers drop out of school to fight scary monsters", "stac_extensions": []}, {"id": "testcollection_2", "type": "Collection", "title": "The Bear", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2022-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": ["restaurant", "funny", "sad", "great"], "description": "Another story about why you should not start a restaurant", "stac_extensions": []}], "numberMatched": 2, "numberReturned": 2} select collection_search('{"q": "large lizard"}'); {"links": [], "collections": [{"id": "testcollection_3", "type": "Collection", "title": "Godzilla", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["1954-01-01T00:00:00+00:00", null]]}, "keywords": ["scary", "lizard", "monster"], "description": "A large lizard takes its revenge", "stac_extensions": []}], "numberMatched": 1, "numberReturned": 1} select collection_search('{"q": "teenagers fight monsters"}'); {"links": [], "collections": [], "numberMatched": 0, "numberReturned": 0} select collection_search('{"q": "scary monsters"}'); {"links": [], "collections": [{"id": "testcollection_1", "type": "Collection", "title": "Stranger Things", "extent": {"spatial": [[-180, -90, 180, 90]], "temporal": [["2016-01-01T00:00:00+00:00", "2025-12-31T23:59:59+00:00"]]}, "keywords": null, "description": "Some teenagers drop out of school to fight scary monsters", "stac_extensions": []}], "numberMatched": 1, "numberReturned": 1} ================================================ FILE: src/pgstac/tests/basic/partitions.sql ================================================ -- run tests as pgstac_ingest SET ROLE pgstac_ingest; SET pgstac.use_queue=FALSE; SELECT get_setting_bool('use_queue'); SET pgstac.update_collection_extent=TRUE; SELECT get_setting_bool('update_collection_extent'); --create base data to use with tests CREATE TEMP TABLE test_items AS SELECT jsonb_build_object( 'id', concat('pgstactest-partitioned-', (row_number() over ())::text), 'collection', 'pgstactest-partitioned', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object( 'datetime', g::text) ) as content FROM generate_series('2020-01-01'::timestamptz, '2022-01-01'::timestamptz, '1 week'::interval) g; --test non-partitioned collection INSERT INTO collections (content) VALUES ('{"id":"pgstactest-partitioned"}'); INSERT INTO items_staging(content) SELECT content FROM test_items; SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned'; --test collection partioned by year INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-year"}', 'year'); INSERT INTO items_staging(content) SELECT content || '{"collection":"pgstactest-partitioned-year"}'::jsonb FROM test_items; SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-year'; --test collection partioned by month INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-month"}', 'month'); INSERT INTO items_staging(content) SELECT content || '{"collection":"pgstactest-partitioned-month"}'::jsonb FROM test_items; SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-month'; --test repartitioning from year to non partitioned UPDATE collections SET partition_trunc=NULL WHERE id='pgstactest-partitioned-year'; SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-year'; SELECT count(*) FROM items WHERE collection='pgstactest-partitioned-year'; --test repartitioning from non-partitioned to year UPDATE collections SET partition_trunc='year' WHERE id='pgstactest-partitioned'; SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned'; SELECT count(*) FROM items WHERE collection='pgstactest-partitioned'; --check that partition stats have been updated SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned' and spatial IS NULL; --test noop for repartitioning UPDATE collections SET content=content || '{"foo":"bar"}'::jsonb WHERE id='pgstactest-partitioned-month'; SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-month'; SELECT count(*) FROM items WHERE collection='pgstactest-partitioned-month'; --test using query queue SET pgstac.use_queue=TRUE; SELECT get_setting_bool('use_queue'); INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-q"}', 'month'); INSERT INTO items_staging(content) SELECT content || '{"collection":"pgstactest-partitioned-q"}'::jsonb FROM test_items; SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q'; --check that partition stats haven't been updated SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q' and spatial IS NULL; --check that queue has items SELECT count(*)>0 FROM query_queue; --run queue items to update partition stats SELECT run_queued_queries_intransaction()>0; --check that queue has been emptied SELECT count(*) FROM query_queue; SELECT run_queued_queries_intransaction(); --check that partition stats have been updated SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q' and spatial IS NULL; --check that collection extents have been updated SELECT id, content->'extent' FROM collections WHERE id LIKE 'pgstactest-partitioned%' ORDER BY id; --check that values for datetimes that are non 4 digit or that have very high precision are ingesting correctly and that partitioning is working for them SET pgstac.use_queue=FALSE; SELECT get_setting_bool('use_queue'); INSERT INTO test_items (content) SELECT jsonb_build_object( 'id', 'pgstactest-partitioned-whackyyear', 'collection', 'pgstactest-partitioned', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object( 'datetime', '10000-01-01T00:00:00Z') ); INSERT INTO test_items (content) SELECT jsonb_build_object( 'id', 'pgstactest-partitioned-whackyprecision', 'collection', 'pgstactest-partitioned', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object( 'datetime', '2000-01-01T00:00:00.12389878917192387129837Z') ); INSERT INTO test_items (content) SELECT jsonb_build_object( 'id', 'pgstactest-partitioned-startend', 'collection', 'pgstactest-partitioned', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object( 'start_datetime', '2000-01-01T00:00:00.12389878917192387129837Z', 'end_datetime', '99999-01-01T00:00:00Z') ); INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-oddballs"}', 'month'); INSERT INTO items_staging(content) SELECT content || '{"collection":"pgstactest-partitioned-oddballs"}'::jsonb FROM test_items; SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-oddballs'; SELECT collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM partitions WHERE collection='pgstactest-partitioned-oddballs' ORDER BY partition_dtrange; ================================================ FILE: src/pgstac/tests/basic/partitions.sql.out ================================================ -- run tests as pgstac_ingest SET ROLE pgstac_ingest; SET SET pgstac.use_queue=FALSE; SET SELECT get_setting_bool('use_queue'); f SET pgstac.update_collection_extent=TRUE; SET SELECT get_setting_bool('update_collection_extent'); t --create base data to use with tests CREATE TEMP TABLE test_items AS SELECT jsonb_build_object( 'id', concat('pgstactest-partitioned-', (row_number() over ())::text), 'collection', 'pgstactest-partitioned', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object( 'datetime', g::text) ) as content FROM generate_series('2020-01-01'::timestamptz, '2022-01-01'::timestamptz, '1 week'::interval) g; SELECT 105 --test non-partitioned collection INSERT INTO collections (content) VALUES ('{"id":"pgstactest-partitioned"}'); INSERT 0 1 INSERT INTO items_staging(content) SELECT content FROM test_items; INSERT 0 105 SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned'; 1 --test collection partioned by year INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-year"}', 'year'); INSERT 0 1 INSERT INTO items_staging(content) SELECT content || '{"collection":"pgstactest-partitioned-year"}'::jsonb FROM test_items; INSERT 0 105 SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-year'; 2 --test collection partioned by month INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-month"}', 'month'); INSERT 0 1 INSERT INTO items_staging(content) SELECT content || '{"collection":"pgstactest-partitioned-month"}'::jsonb FROM test_items; INSERT 0 105 SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-month'; 24 --test repartitioning from year to non partitioned UPDATE collections SET partition_trunc=NULL WHERE id='pgstactest-partitioned-year'; UPDATE 1 SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-year'; 1 SELECT count(*) FROM items WHERE collection='pgstactest-partitioned-year'; 105 --test repartitioning from non-partitioned to year UPDATE collections SET partition_trunc='year' WHERE id='pgstactest-partitioned'; UPDATE 1 SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned'; 2 SELECT count(*) FROM items WHERE collection='pgstactest-partitioned'; 105 --check that partition stats have been updated SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned' and spatial IS NULL; 0 --test noop for repartitioning UPDATE collections SET content=content || '{"foo":"bar"}'::jsonb WHERE id='pgstactest-partitioned-month'; UPDATE 1 SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-month'; 24 SELECT count(*) FROM items WHERE collection='pgstactest-partitioned-month'; 105 --test using query queue SET pgstac.use_queue=TRUE; SET SELECT get_setting_bool('use_queue'); t INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-q"}', 'month'); INSERT 0 1 INSERT INTO items_staging(content) SELECT content || '{"collection":"pgstactest-partitioned-q"}'::jsonb FROM test_items; INSERT 0 105 SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q'; 24 --check that partition stats haven't been updated SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q' and spatial IS NULL; 24 --check that queue has items SELECT count(*)>0 FROM query_queue; t --run queue items to update partition stats SELECT run_queued_queries_intransaction()>0; t --check that queue has been emptied SELECT count(*) FROM query_queue; 0 SELECT run_queued_queries_intransaction(); 0 --check that partition stats have been updated SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-q' and spatial IS NULL; 0 --check that collection extents have been updated SELECT id, content->'extent' FROM collections WHERE id LIKE 'pgstactest-partitioned%' ORDER BY id; pgstactest-partitioned | {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} pgstactest-partitioned-month | {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} pgstactest-partitioned-q | {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} pgstactest-partitioned-year | {"spatial": {"bbox": [[-85.3792495727539, 30.933948516845703, -85.30819702148438, 31.003555297851562]]}, "temporal": {"interval": [["2020-01-01T00:00:00+00:00", "2021-12-29T00:00:00+00:00"]]}} --check that values for datetimes that are non 4 digit or that have very high precision are ingesting correctly and that partitioning is working for them SET pgstac.use_queue=FALSE; SET SELECT get_setting_bool('use_queue'); f INSERT INTO test_items (content) SELECT jsonb_build_object( 'id', 'pgstactest-partitioned-whackyyear', 'collection', 'pgstactest-partitioned', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object( 'datetime', '10000-01-01T00:00:00Z') ); INSERT 0 1 INSERT INTO test_items (content) SELECT jsonb_build_object( 'id', 'pgstactest-partitioned-whackyprecision', 'collection', 'pgstactest-partitioned', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object( 'datetime', '2000-01-01T00:00:00.12389878917192387129837Z') ); INSERT 0 1 INSERT INTO test_items (content) SELECT jsonb_build_object( 'id', 'pgstactest-partitioned-startend', 'collection', 'pgstactest-partitioned', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object( 'start_datetime', '2000-01-01T00:00:00.12389878917192387129837Z', 'end_datetime', '99999-01-01T00:00:00Z') ); INSERT 0 1 INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-partitioned-oddballs"}', 'month'); INSERT 0 1 INSERT INTO items_staging(content) SELECT content || '{"collection":"pgstactest-partitioned-oddballs"}'::jsonb FROM test_items; INSERT 0 108 SELECT count(*) FROM partitions WHERE collection='pgstactest-partitioned-oddballs'; 26 SELECT collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM partitions WHERE collection='pgstactest-partitioned-oddballs' ORDER BY partition_dtrange; pgstactest-partitioned-oddballs | ["2000-01-01 00:00:00+00","2000-02-01 00:00:00+00") | ["2000-01-01 00:00:00+00","2000-02-01 00:00:00+00") | [-infinity,infinity] | ["2000-01-01 00:00:00.123899+00","2000-01-01 00:00:00.123899+00"] | ["2000-01-01 00:00:00.123899+00","99999-01-01 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-01-01 00:00:00+00","2020-02-01 00:00:00+00") | ["2020-01-01 00:00:00+00","2020-02-01 00:00:00+00") | [-infinity,infinity] | ["2020-01-01 00:00:00+00","2020-01-29 00:00:00+00"] | ["2020-01-01 00:00:00+00","2020-01-29 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-02-01 00:00:00+00","2020-03-01 00:00:00+00") | ["2020-02-01 00:00:00+00","2020-03-01 00:00:00+00") | [-infinity,infinity] | ["2020-02-05 00:00:00+00","2020-02-26 00:00:00+00"] | ["2020-02-05 00:00:00+00","2020-02-26 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-03-01 00:00:00+00","2020-04-01 00:00:00+00") | ["2020-03-01 00:00:00+00","2020-04-01 00:00:00+00") | [-infinity,infinity] | ["2020-03-04 00:00:00+00","2020-03-25 00:00:00+00"] | ["2020-03-04 00:00:00+00","2020-03-25 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-04-01 00:00:00+00","2020-05-01 00:00:00+00") | ["2020-04-01 00:00:00+00","2020-05-01 00:00:00+00") | [-infinity,infinity] | ["2020-04-01 00:00:00+00","2020-04-29 00:00:00+00"] | ["2020-04-01 00:00:00+00","2020-04-29 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-05-01 00:00:00+00","2020-06-01 00:00:00+00") | ["2020-05-01 00:00:00+00","2020-06-01 00:00:00+00") | [-infinity,infinity] | ["2020-05-06 00:00:00+00","2020-05-27 00:00:00+00"] | ["2020-05-06 00:00:00+00","2020-05-27 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-06-01 00:00:00+00","2020-07-01 00:00:00+00") | ["2020-06-01 00:00:00+00","2020-07-01 00:00:00+00") | [-infinity,infinity] | ["2020-06-03 00:00:00+00","2020-06-24 00:00:00+00"] | ["2020-06-03 00:00:00+00","2020-06-24 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-07-01 00:00:00+00","2020-08-01 00:00:00+00") | ["2020-07-01 00:00:00+00","2020-08-01 00:00:00+00") | [-infinity,infinity] | ["2020-07-01 00:00:00+00","2020-07-29 00:00:00+00"] | ["2020-07-01 00:00:00+00","2020-07-29 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-08-01 00:00:00+00","2020-09-01 00:00:00+00") | ["2020-08-01 00:00:00+00","2020-09-01 00:00:00+00") | [-infinity,infinity] | ["2020-08-05 00:00:00+00","2020-08-26 00:00:00+00"] | ["2020-08-05 00:00:00+00","2020-08-26 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-09-01 00:00:00+00","2020-10-01 00:00:00+00") | ["2020-09-01 00:00:00+00","2020-10-01 00:00:00+00") | [-infinity,infinity] | ["2020-09-02 00:00:00+00","2020-09-30 00:00:00+00"] | ["2020-09-02 00:00:00+00","2020-09-30 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-10-01 00:00:00+00","2020-11-01 00:00:00+00") | ["2020-10-01 00:00:00+00","2020-11-01 00:00:00+00") | [-infinity,infinity] | ["2020-10-07 00:00:00+00","2020-10-28 00:00:00+00"] | ["2020-10-07 00:00:00+00","2020-10-28 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-11-01 00:00:00+00","2020-12-01 00:00:00+00") | ["2020-11-01 00:00:00+00","2020-12-01 00:00:00+00") | [-infinity,infinity] | ["2020-11-04 00:00:00+00","2020-11-25 00:00:00+00"] | ["2020-11-04 00:00:00+00","2020-11-25 00:00:00+00"] pgstactest-partitioned-oddballs | ["2020-12-01 00:00:00+00","2021-01-01 00:00:00+00") | ["2020-12-01 00:00:00+00","2021-01-01 00:00:00+00") | [-infinity,infinity] | ["2020-12-02 00:00:00+00","2020-12-30 00:00:00+00"] | ["2020-12-02 00:00:00+00","2020-12-30 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-01-01 00:00:00+00","2021-02-01 00:00:00+00") | ["2021-01-01 00:00:00+00","2021-02-01 00:00:00+00") | [-infinity,infinity] | ["2021-01-06 00:00:00+00","2021-01-27 00:00:00+00"] | ["2021-01-06 00:00:00+00","2021-01-27 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-02-01 00:00:00+00","2021-03-01 00:00:00+00") | ["2021-02-01 00:00:00+00","2021-03-01 00:00:00+00") | [-infinity,infinity] | ["2021-02-03 00:00:00+00","2021-02-24 00:00:00+00"] | ["2021-02-03 00:00:00+00","2021-02-24 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-03-01 00:00:00+00","2021-04-01 00:00:00+00") | ["2021-03-01 00:00:00+00","2021-04-01 00:00:00+00") | [-infinity,infinity] | ["2021-03-03 00:00:00+00","2021-03-31 00:00:00+00"] | ["2021-03-03 00:00:00+00","2021-03-31 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-04-01 00:00:00+00","2021-05-01 00:00:00+00") | ["2021-04-01 00:00:00+00","2021-05-01 00:00:00+00") | [-infinity,infinity] | ["2021-04-07 00:00:00+00","2021-04-28 00:00:00+00"] | ["2021-04-07 00:00:00+00","2021-04-28 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-05-01 00:00:00+00","2021-06-01 00:00:00+00") | ["2021-05-01 00:00:00+00","2021-06-01 00:00:00+00") | [-infinity,infinity] | ["2021-05-05 00:00:00+00","2021-05-26 00:00:00+00"] | ["2021-05-05 00:00:00+00","2021-05-26 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-06-01 00:00:00+00","2021-07-01 00:00:00+00") | ["2021-06-01 00:00:00+00","2021-07-01 00:00:00+00") | [-infinity,infinity] | ["2021-06-02 00:00:00+00","2021-06-30 00:00:00+00"] | ["2021-06-02 00:00:00+00","2021-06-30 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-07-01 00:00:00+00","2021-08-01 00:00:00+00") | ["2021-07-01 00:00:00+00","2021-08-01 00:00:00+00") | [-infinity,infinity] | ["2021-07-07 00:00:00+00","2021-07-28 00:00:00+00"] | ["2021-07-07 00:00:00+00","2021-07-28 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-08-01 00:00:00+00","2021-09-01 00:00:00+00") | ["2021-08-01 00:00:00+00","2021-09-01 00:00:00+00") | [-infinity,infinity] | ["2021-08-04 00:00:00+00","2021-08-25 00:00:00+00"] | ["2021-08-04 00:00:00+00","2021-08-25 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-09-01 00:00:00+00","2021-10-01 00:00:00+00") | ["2021-09-01 00:00:00+00","2021-10-01 00:00:00+00") | [-infinity,infinity] | ["2021-09-01 00:00:00+00","2021-09-29 00:00:00+00"] | ["2021-09-01 00:00:00+00","2021-09-29 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-10-01 00:00:00+00","2021-11-01 00:00:00+00") | ["2021-10-01 00:00:00+00","2021-11-01 00:00:00+00") | [-infinity,infinity] | ["2021-10-06 00:00:00+00","2021-10-27 00:00:00+00"] | ["2021-10-06 00:00:00+00","2021-10-27 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-11-01 00:00:00+00","2021-12-01 00:00:00+00") | ["2021-11-01 00:00:00+00","2021-12-01 00:00:00+00") | [-infinity,infinity] | ["2021-11-03 00:00:00+00","2021-11-24 00:00:00+00"] | ["2021-11-03 00:00:00+00","2021-11-24 00:00:00+00"] pgstactest-partitioned-oddballs | ["2021-12-01 00:00:00+00","2022-01-01 00:00:00+00") | ["2021-12-01 00:00:00+00","2022-01-01 00:00:00+00") | [-infinity,infinity] | ["2021-12-01 00:00:00+00","2021-12-29 00:00:00+00"] | ["2021-12-01 00:00:00+00","2021-12-29 00:00:00+00"] pgstactest-partitioned-oddballs | ["10000-01-01 00:00:00+00","10000-02-01 00:00:00+00") | ["10000-01-01 00:00:00+00","10000-02-01 00:00:00+00") | [-infinity,infinity] | ["10000-01-01 00:00:00+00","10000-01-01 00:00:00+00"] | ["10000-01-01 00:00:00+00","10000-01-01 00:00:00+00"] ================================================ FILE: src/pgstac/tests/basic/search_path.sql ================================================ -- Test that partition views and tables work identically with and without pgstac in search_path SET ROLE pgstac_ingest; SET pgstac.use_queue=FALSE; -- Set up test data with pgstac in search_path INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-searchpath"}', 'month'); CREATE TEMP TABLE sp_test_items AS SELECT jsonb_build_object( 'id', concat('pgstactest-searchpath-', (row_number() over ())::text), 'collection', 'pgstactest-searchpath', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object('datetime', g::text) ) as content FROM generate_series('2020-01-01'::timestamptz, '2020-06-01'::timestamptz, '1 week'::interval) g; INSERT INTO items_staging(content) SELECT content FROM sp_test_items; -- Capture results with pgstac in search_path CREATE TEMP TABLE sp_partition_sys_meta AS SELECT * FROM partition_sys_meta WHERE collection='pgstactest-searchpath' ORDER BY partition; CREATE TEMP TABLE sp_partition_stats AS SELECT * FROM partition_stats WHERE partition IN (SELECT partition FROM sp_partition_sys_meta) ORDER BY partition; CREATE TEMP TABLE sp_partitions AS SELECT * FROM partitions WHERE collection='pgstactest-searchpath' ORDER BY partition; CREATE TEMP TABLE sp_partitions_view AS SELECT * FROM partitions_view WHERE collection='pgstactest-searchpath' ORDER BY partition; CREATE TEMP TABLE sp_partition_steps AS SELECT * FROM partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) ORDER BY name; -- Verify we have data SELECT count(*) > 0 AS has_partition_sys_meta FROM sp_partition_sys_meta; SELECT count(*) > 0 AS has_partition_stats FROM sp_partition_stats; SELECT count(*) > 0 AS has_partitions FROM sp_partitions; SELECT count(*) > 0 AS has_partitions_view FROM sp_partitions_view; SELECT count(*) > 0 AS has_partition_steps FROM sp_partition_steps; -- Now remove pgstac from search_path SET search_path TO public; -- partition_sys_meta: compare counts and key columns SELECT ( SELECT count(*) FROM pgstac.partition_sys_meta WHERE collection='pgstactest-searchpath' ) = ( SELECT count(*) FROM sp_partition_sys_meta ) AS partition_sys_meta_count_match; SELECT count(*) = 0 AS partition_sys_meta_data_match FROM ( (SELECT partition, collection, constraint_dtrange, constraint_edtrange FROM pgstac.partition_sys_meta WHERE collection='pgstactest-searchpath' EXCEPT SELECT partition, collection, constraint_dtrange, constraint_edtrange FROM sp_partition_sys_meta) UNION ALL (SELECT partition, collection, constraint_dtrange, constraint_edtrange FROM sp_partition_sys_meta EXCEPT SELECT partition, collection, constraint_dtrange, constraint_edtrange FROM pgstac.partition_sys_meta WHERE collection='pgstactest-searchpath') ) diff; -- partition_stats: compare counts and key columns SELECT ( SELECT count(*) FROM pgstac.partition_stats WHERE partition IN (SELECT partition FROM sp_partition_sys_meta) ) = ( SELECT count(*) FROM sp_partition_stats ) AS partition_stats_count_match; SELECT count(*) = 0 AS partition_stats_data_match FROM ( (SELECT partition, dtrange, edtrange FROM pgstac.partition_stats WHERE partition IN (SELECT partition FROM sp_partition_sys_meta) EXCEPT SELECT partition, dtrange, edtrange FROM sp_partition_stats) UNION ALL (SELECT partition, dtrange, edtrange FROM sp_partition_stats EXCEPT SELECT partition, dtrange, edtrange FROM pgstac.partition_stats WHERE partition IN (SELECT partition FROM sp_partition_sys_meta)) ) diff; -- partitions (materialized view): compare counts and key columns SELECT ( SELECT count(*) FROM pgstac.partitions WHERE collection='pgstactest-searchpath' ) = ( SELECT count(*) FROM sp_partitions ) AS partitions_count_match; SELECT count(*) = 0 AS partitions_data_match FROM ( (SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM pgstac.partitions WHERE collection='pgstactest-searchpath' EXCEPT SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM sp_partitions) UNION ALL (SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM sp_partitions EXCEPT SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM pgstac.partitions WHERE collection='pgstactest-searchpath') ) diff; -- partitions_view: compare counts and key columns SELECT ( SELECT count(*) FROM pgstac.partitions_view WHERE collection='pgstactest-searchpath' ) = ( SELECT count(*) FROM sp_partitions_view ) AS partitions_view_count_match; SELECT count(*) = 0 AS partitions_view_data_match FROM ( (SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM pgstac.partitions_view WHERE collection='pgstactest-searchpath' EXCEPT SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM sp_partitions_view) UNION ALL (SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM sp_partitions_view EXCEPT SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM pgstac.partitions_view WHERE collection='pgstactest-searchpath') ) diff; -- partition_steps: compare counts and key columns SELECT ( SELECT count(*) FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) ) = ( SELECT count(*) FROM sp_partition_steps ) AS partition_steps_count_match; SELECT count(*) = 0 AS partition_steps_data_match FROM ( (SELECT name, sdate, edate FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) EXCEPT SELECT name, sdate, edate FROM sp_partition_steps) UNION ALL (SELECT name, sdate, edate FROM sp_partition_steps EXCEPT SELECT name, sdate, edate FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta)) ) diff; -- Restore search_path SET search_path TO pgstac, public; ================================================ FILE: src/pgstac/tests/basic/search_path.sql.out ================================================ -- Test that partition views and tables work identically with and without pgstac in search_path SET ROLE pgstac_ingest; SET SET pgstac.use_queue=FALSE; SET -- Set up test data with pgstac in search_path INSERT INTO collections (content, partition_trunc) VALUES ('{"id":"pgstactest-searchpath"}', 'month'); INSERT 0 1 CREATE TEMP TABLE sp_test_items AS SELECT jsonb_build_object( 'id', concat('pgstactest-searchpath-', (row_number() over ())::text), 'collection', 'pgstactest-searchpath', 'geometry', '{"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}'::json, 'properties', jsonb_build_object('datetime', g::text) ) as content FROM generate_series('2020-01-01'::timestamptz, '2020-06-01'::timestamptz, '1 week'::interval) g; SELECT 22 INSERT INTO items_staging(content) SELECT content FROM sp_test_items; INSERT 0 22 -- Capture results with pgstac in search_path CREATE TEMP TABLE sp_partition_sys_meta AS SELECT * FROM partition_sys_meta WHERE collection='pgstactest-searchpath' ORDER BY partition; SELECT 5 CREATE TEMP TABLE sp_partition_stats AS SELECT * FROM partition_stats WHERE partition IN (SELECT partition FROM sp_partition_sys_meta) ORDER BY partition; SELECT 5 CREATE TEMP TABLE sp_partitions AS SELECT * FROM partitions WHERE collection='pgstactest-searchpath' ORDER BY partition; SELECT 5 CREATE TEMP TABLE sp_partitions_view AS SELECT * FROM partitions_view WHERE collection='pgstactest-searchpath' ORDER BY partition; SELECT 5 CREATE TEMP TABLE sp_partition_steps AS SELECT * FROM partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) ORDER BY name; SELECT 5 -- Verify we have data SELECT count(*) > 0 AS has_partition_sys_meta FROM sp_partition_sys_meta; t SELECT count(*) > 0 AS has_partition_stats FROM sp_partition_stats; t SELECT count(*) > 0 AS has_partitions FROM sp_partitions; t SELECT count(*) > 0 AS has_partitions_view FROM sp_partitions_view; t SELECT count(*) > 0 AS has_partition_steps FROM sp_partition_steps; t -- Now remove pgstac from search_path SET search_path TO public; SET -- partition_sys_meta: compare counts and key columns SELECT ( SELECT count(*) FROM pgstac.partition_sys_meta WHERE collection='pgstactest-searchpath' ) = ( SELECT count(*) FROM sp_partition_sys_meta ) AS partition_sys_meta_count_match; t SELECT count(*) = 0 AS partition_sys_meta_data_match FROM ( (SELECT partition, collection, constraint_dtrange, constraint_edtrange FROM pgstac.partition_sys_meta WHERE collection='pgstactest-searchpath' EXCEPT SELECT partition, collection, constraint_dtrange, constraint_edtrange FROM sp_partition_sys_meta) UNION ALL (SELECT partition, collection, constraint_dtrange, constraint_edtrange FROM sp_partition_sys_meta EXCEPT SELECT partition, collection, constraint_dtrange, constraint_edtrange FROM pgstac.partition_sys_meta WHERE collection='pgstactest-searchpath') ) diff; t -- partition_stats: compare counts and key columns SELECT ( SELECT count(*) FROM pgstac.partition_stats WHERE partition IN (SELECT partition FROM sp_partition_sys_meta) ) = ( SELECT count(*) FROM sp_partition_stats ) AS partition_stats_count_match; t SELECT count(*) = 0 AS partition_stats_data_match FROM ( (SELECT partition, dtrange, edtrange FROM pgstac.partition_stats WHERE partition IN (SELECT partition FROM sp_partition_sys_meta) EXCEPT SELECT partition, dtrange, edtrange FROM sp_partition_stats) UNION ALL (SELECT partition, dtrange, edtrange FROM sp_partition_stats EXCEPT SELECT partition, dtrange, edtrange FROM pgstac.partition_stats WHERE partition IN (SELECT partition FROM sp_partition_sys_meta)) ) diff; t -- partitions (materialized view): compare counts and key columns SELECT ( SELECT count(*) FROM pgstac.partitions WHERE collection='pgstactest-searchpath' ) = ( SELECT count(*) FROM sp_partitions ) AS partitions_count_match; t SELECT count(*) = 0 AS partitions_data_match FROM ( (SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM pgstac.partitions WHERE collection='pgstactest-searchpath' EXCEPT SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM sp_partitions) UNION ALL (SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM sp_partitions EXCEPT SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM pgstac.partitions WHERE collection='pgstactest-searchpath') ) diff; t -- partitions_view: compare counts and key columns SELECT ( SELECT count(*) FROM pgstac.partitions_view WHERE collection='pgstactest-searchpath' ) = ( SELECT count(*) FROM sp_partitions_view ) AS partitions_view_count_match; t SELECT count(*) = 0 AS partitions_view_data_match FROM ( (SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM pgstac.partitions_view WHERE collection='pgstactest-searchpath' EXCEPT SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM sp_partitions_view) UNION ALL (SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM sp_partitions_view EXCEPT SELECT partition, collection, partition_dtrange, constraint_dtrange, constraint_edtrange, dtrange, edtrange FROM pgstac.partitions_view WHERE collection='pgstactest-searchpath') ) diff; t -- partition_steps: compare counts and key columns SELECT ( SELECT count(*) FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) ) = ( SELECT count(*) FROM sp_partition_steps ) AS partition_steps_count_match; t SELECT count(*) = 0 AS partition_steps_data_match FROM ( (SELECT name, sdate, edate FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta) EXCEPT SELECT name, sdate, edate FROM sp_partition_steps) UNION ALL (SELECT name, sdate, edate FROM sp_partition_steps EXCEPT SELECT name, sdate, edate FROM pgstac.partition_steps WHERE name IN (SELECT partition FROM sp_partition_sys_meta)) ) diff; t -- Restore search_path SET search_path TO pgstac, public; SET ================================================ FILE: src/pgstac/tests/basic/xyz_searches.sql ================================================ SET pgstac."default_filter_lang" TO 'cql-json'; SELECT hash from search_query('{"collections":["pgstac-test-collection"]}'); SELECT hash, search, metadata FROM search_fromhash('2bbae9a0ef0bbb5ffaca06603ce621d7'); SELECT xyzsearch(8615, 13418, 15, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb); SELECT xyzsearch(1048, 1682, 12, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb); SELECT xyzsearch(1048, 1682, 12, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, NULL, 1); SELECT xyzsearch(16792, 26892, 16, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, exitwhenfull => true); SELECT xyzsearch(16792, 26892, 16, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, exitwhenfull => false, skipcovered => false); SELECT geojsonsearch('{"type": "Point","coordinates": [-87.75608539581299,30.692471153735646]}', '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, exitwhenfull => true, skipcovered => true); SELECT geojsonsearch('{"type": "Point","coordinates": [-87.75608539581299,30.692471153735646]}', '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, exitwhenfull => false, skipcovered => false) s; ================================================ FILE: src/pgstac/tests/basic/xyz_searches.sql.out ================================================ SET pgstac."default_filter_lang" TO 'cql-json'; SET SELECT hash from search_query('{"collections":["pgstac-test-collection"]}'); 2bbae9a0ef0bbb5ffaca06603ce621d7 SELECT hash, search, metadata FROM search_fromhash('2bbae9a0ef0bbb5ffaca06603ce621d7'); 2bbae9a0ef0bbb5ffaca06603ce621d7 | {"collections": ["pgstac-test-collection"]} | {} SELECT xyzsearch(8615, 13418, 15, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb); {"type": "FeatureCollection", "features": [{"id": "pgstac-test-item-0003", "collection": "pgstac-test-collection"}]} SELECT xyzsearch(1048, 1682, 12, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb); {"type": "FeatureCollection", "features": [{"id": "pgstac-test-item-0050", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0049", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0048", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0047", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0100", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0089", "collection": "pgstac-test-collection"}]} SELECT xyzsearch(1048, 1682, 12, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, NULL, 1); {"type": "FeatureCollection", "features": [{"id": "pgstac-test-item-0050", "collection": "pgstac-test-collection"}]} SELECT xyzsearch(16792, 26892, 16, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, exitwhenfull => true); {"type": "FeatureCollection", "features": [{"id": "pgstac-test-item-0098", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}]} SELECT xyzsearch(16792, 26892, 16, '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, exitwhenfull => false, skipcovered => false); {"type": "FeatureCollection", "features": [{"id": "pgstac-test-item-0098", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}, {"id": "pgstac-test-item-0091", "collection": "pgstac-test-collection"}]} SELECT geojsonsearch('{"type": "Point","coordinates": [-87.75608539581299,30.692471153735646]}', '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, exitwhenfull => true, skipcovered => true); {"type": "FeatureCollection", "features": [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}]} SELECT geojsonsearch('{"type": "Point","coordinates": [-87.75608539581299,30.692471153735646]}', '2bbae9a0ef0bbb5ffaca06603ce621d7', '{"include":["id"]}'::jsonb, exitwhenfull => false, skipcovered => false) s; {"type": "FeatureCollection", "features": [{"id": "pgstac-test-item-0097", "collection": "pgstac-test-collection"}]} ================================================ FILE: src/pgstac/tests/pgtap/001_core.sql ================================================ -- Check that schema exists SELECT has_schema('pgstac'::name); -- Check that PostGIS extension are installed and available on the path SELECT has_extension('postgis'); SELECT has_table('pgstac'::name, 'migrations'::name); SELECT has_function('pgstac'::name, 'to_text_array', ARRAY['jsonb']); SELECT results_eq( $$ SELECT to_text_array('["a","b","c"]'::jsonb) $$, $$ SELECT '{a,b,c}'::text[] $$, 'to_text_array returns text[] from jsonb array' ); SET pgstac.readonly to 'false'; SELECT results_eq( $$ SELECT pgstac.readonly(); $$, $$ SELECT FALSE; $$, 'Readonly is set to false' ); SELECT lives_ok( $$ SELECT search('{}'); $$, 'Search works with readonly mode set to off in readwrite mode.' ); RESET pgstac.context; SELECT is_definer('update_partition_stats'); SELECT is_definer('partition_after_triggerfunc'); SELECT is_definer('drop_table_constraints'); SELECT is_definer('create_table_constraints'); SELECT is_definer('check_partition'); SELECT is_definer('repartition'); SELECT is_definer('where_stats'); SELECT is_definer('search_query'); SELECT is_definer('format_item'); SELECT is_definer('maintain_index'); ================================================ FILE: src/pgstac/tests/pgtap/001a_jsonutils.sql ================================================ SELECT has_function('pgstac'::name, 'to_text_array', ARRAY['jsonb']); SELECT results_eq( $$ SELECT to_text_array('["a","b","c"]'::jsonb) $$, $$ SELECT '{a,b,c}'::text[] $$, 'textarr returns text[] from jsonb array' ); ================================================ FILE: src/pgstac/tests/pgtap/001b_cursorutils.sql ================================================ ================================================ FILE: src/pgstac/tests/pgtap/001s_stacutils.sql ================================================ SELECT has_function('pgstac'::name, 'stac_geom', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'stac_datetime', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'stac_end_datetime', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'stac_daterange', ARRAY['jsonb']); ================================================ FILE: src/pgstac/tests/pgtap/002_collections.sql ================================================ SELECT has_table('pgstac'::name, 'collections'::name); SELECT col_is_pk('pgstac'::name, 'collections'::name, 'key', 'collections has primary key'); SELECT has_function('pgstac'::name, 'create_collection', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'update_collection', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'upsert_collection', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'get_collection', ARRAY['text']); SELECT has_function('pgstac'::name, 'delete_collection', ARRAY['text']); SELECT has_function('pgstac'::name, 'all_collections', '{}'::text[]); DELETE FROM collections WHERE id in ('pgstac-test-collection', 'pgstac-test-collection2'); \copy collections (content) FROM 'tests/testdata/collections.ndjson'; \copy items_staging (content) FROM 'tests/testdata/items.ndjson'; SELECT results_eq($$ SELECT count(*) FROM partitions_view WHERE collection='pgstac-test-collection'; $$, $$ SELECT 1::bigint $$, 'Test that partition metadata only has one record after adding items data for one collection' ); SELECT results_eq($$ SELECT count(*) FROM partitions_view WHERE collection='pgstac-test-collection2'; $$, $$ SELECT 0::bigint $$, 'Test that partition metadata does not have collection 2 yet' ); SELECT create_item('{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection2", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}'); SELECT results_eq($$ SELECT count(*) FROM partitions_view WHERE collection='pgstac-test-collection2'; $$, $$ SELECT 1::bigint $$, 'Test that partition metadata only has record for collection 2' ); DELETE FROM collections WHERE id='pgstac-test-collection2'; SELECT results_eq($$ SELECT count(*) FROM partition_sys_meta WHERE collection='pgstac-test-collection2'; $$, $$ SELECT 0::bigint $$, 'Test that sys meta does not have for collection 2 record after removing collection 2' ); DELETE FROM collections WHERE id='pgstac-test-collection'; SELECT results_eq($$ SELECT count(*) FROM partition_sys_meta WHERE collection='pgstac-test-collection'; $$, $$ SELECT 0::bigint $$, 'Test that sys meta does not have for collection 1 record after removing collection 1' ); \copy collections (content) FROM 'tests/testdata/collections.ndjson'; \copy items_staging (content) FROM 'tests/testdata/items.ndjson'; SELECT results_eq($$ SELECT count(*) FROM partitions_view WHERE collection='pgstac-test-collection'; $$, $$ SELECT 1::bigint $$, 'Test that partition metadata only has one record after adding items data for one collection' ); SELECT results_eq($$ SELECT count(*) FROM partitions_view WHERE collection='pgstac-test-collection2'; $$, $$ SELECT 0::bigint $$, 'Test that partition metadata does not have collection 2 yet' ); SELECT create_item('{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection2", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}'); SELECT results_eq($$ SELECT count(*) FROM partitions_view WHERE collection='pgstac-test-collection2'; $$, $$ SELECT 1::bigint $$, 'Test that partition metadata only has record for collection 2' ); ================================================ FILE: src/pgstac/tests/pgtap/002a_queryables.sql ================================================ SELECT results_eq( $$ SELECT sort_sqlorderby('{"sortby":{"field":"properties.eo:cloud_cover"}}'); $$, $$ SELECT sort_sqlorderby('{"sortby":{"field":"eo:cloud_cover"}}'); $$, 'Make sure that sortby with/without properties prefix return the same sort statement.' ); SET pgstac."default_filter_lang" TO 'cql2-json'; SELECT results_eq( $$ SELECT stac_search_to_where('{"filter":{"op":"eq","args":[{"property":"eo:cloud_cover"},0]}}'); $$, $$ SELECT stac_search_to_where('{"filter":{"op":"eq","args":[{"property":"properties.eo:cloud_cover"},0]}}'); $$, 'Make sure that CQL2 filter works the same with/without properties prefix.' ); SET pgstac."default_filter_lang" TO 'cql-json'; SELECT results_eq( $$ SELECT stac_search_to_where('{"filter":{"eq":[{"property":"eo:cloud_cover"},0]}}'); $$, $$ SELECT stac_search_to_where('{"filter":{"eq":[{"property":"properties.eo:cloud_cover"},0]}}'); $$, 'Make sure that CQL filter works the same with/without properties prefix.' ); DELETE FROM collections WHERE id in ('pgstac-test-collection', 'pgstac-test-collection2'); SELECT results_eq( $$ SELECT all_collections(); $$, $$ SELECT '[]'::jsonb; $$, 'Make sure all_collections returns an empty array when the collection table is empty.' ); \copy collections (content) FROM 'tests/testdata/collections.ndjson'; SELECT results_eq( $$ SELECT get_queryables('pgstac-test-collection') -> 'properties' ? 'datetime'; $$, $$ SELECT true; $$, 'Make sure valid schema object is returned for a existing collection.' ); SELECT results_eq( $$ SELECT get_queryables('foo'); $$, $$ SELECT NULL::jsonb; $$, 'Make sure null is returned for a non-existant collection.' ); SELECT lives_ok( $$ DELETE FROM queryables WHERE name IN ('testqueryable', 'testqueryable2', 'testqueryable3'); $$, 'Make sure test queryable does not exist.' ); SELECT lives_ok( $$ INSERT INTO queryables (name, collection_ids) VALUES ('testqueryable', null); $$, 'Can add a new queryable that applies to all collections.' ); select is( (SELECT count(*) from collections where id = 'pgstac-test-collection'), '1', 'Make sure test collection exists.' ); SELECT lives_ok( $$ INSERT INTO queryables (name, collection_ids) VALUES ('testqueryable3', '{pgstac-test-collection}'); $$, 'Can add a new queryable to a specific existing collection.' ); SELECT throws_ok( $$ INSERT INTO queryables (name, collection_ids) VALUES ('testqueryable2', '{nonexistent}'); $$, '23503' ); SELECT throws_ok( $$ INSERT INTO queryables (name, collection_ids) VALUES ('testqueryable', '{pgstac-test-collection}'); $$, '23505' ); SELECT lives_ok( $$ UPDATE queryables SET collection_ids = '{pgstac-test-collection}' WHERE name='testqueryable'; $$, 'Can update a queryable from null to a single collection.' ); SET pgstac.additional_properties to 'false'; SELECT results_eq( $$ SELECT pgstac.additional_properties(); $$, $$ SELECT FALSE; $$, 'Make sure additional_properties is set to false' ); SELECT throws_ok( $$ SELECT search('{"filter": {"eq": [{"property": "xyzzy"}, "dummy"]}}'); $$, 'Term xyzzy is not found in queryables.', 'Make sure a term not present in the list of queryables cannot be used in a filter' ); SELECT lives_ok( $$ SELECT search('{"filter": {"eq": [{"property": "datetime"}, "2020-11-11T00:00:00Z"]}}'); $$, 'Make sure a term present in the list of queryables can be used in a filter' ); SELECT lives_ok( $$ SELECT search('{"filter": {"s_intersects": [{"property": "geometry"}, {"type": "Point", "coordinates": [0, 0]}]}}'); $$, 'Make sure a term present in the list of queryables can be used in a filter' ); SELECT lives_ok( $$ SELECT search('{"filter": {"and": [{"t_after": [{"property": "datetime"}, "2020-11-11T00:00:00"]}, {"t_before": [{"property": "datetime"}, "2022-11-11T00:00:00"]}]}}'); $$, 'Make sure that only arguments that are properties are checked' ); SELECT throws_ok( $$ SELECT search('{"filter": {"and": [{"t_after": [{"property": "datetime"}, "2020-11-11T00:00:00"]}, {"eq": [{"property": "xyzzy"}, "dummy"]}]}}'); $$, 'Term xyzzy is not found in queryables.', 'Make sure a term not present in the list of queryables cannot be used in a filter with nested arguments' ); SET pgstac.additional_properties to 'true'; SELECT results_eq( $$ SELECT pgstac.additional_properties(); $$, $$ SELECT TRUE; $$, 'Make sure additional_properties is set to true' ); SELECT lives_ok( $$ SELECT search('{"filter": {"eq": [{"property": "xyzzy"}, "dummy"]}}'); $$, 'Make sure a term not present in the list of queryables can be used in a filter' ); SELECT lives_ok( $$ SELECT search('{"filter": {"eq": [{"property": "datetime"}, "2020-11-11T00:00:00Z"]}}'); $$, 'Make sure a term present in the list of queryables can be used in a filter' ); RESET pgstac.additional_properties; ================================================ FILE: src/pgstac/tests/pgtap/003_items.sql ================================================ SELECT has_table('pgstac'::name, 'items'::name); SELECT is_indexed('pgstac'::name, 'items'::name, 'geometry'); SELECT is_partitioned('pgstac'::name,'items'::name); SELECT has_function('pgstac'::name, 'get_item', ARRAY['text','text']); SELECT has_function('pgstac'::name, 'delete_item', ARRAY['text','text']); SELECT has_function('pgstac'::name, 'create_item', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'update_item', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'upsert_item', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'create_items', ARRAY['jsonb']); SELECT has_function('pgstac'::name, 'upsert_items', ARRAY['jsonb']); -- tools to update collection extents based on extents in items SELECT has_function('pgstac'::name, 'collection_bbox', ARRAY['text']); SELECT has_function('pgstac'::name, 'collection_temporal_extent', ARRAY['text']); SELECT has_function('pgstac'::name, 'update_collection_extents', '{}'::text[]); DELETE FROM collections WHERE id in ('pgstac-test-collection', 'pgstac-test-collection2'); \copy collections (content) FROM 'tests/testdata/collections.ndjson'; SELECT create_item('{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}'); SELECT results_eq($$ SELECT content->'properties'->>'eo:cloud_cover' FROM items WHERE collection='pgstac-test-collection'; $$,$$ SELECT '28'; $$, 'Test create_item function' ); SELECT update_item('{"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 29, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]}'); SELECT results_eq($$ SELECT content->'properties'->>'eo:cloud_cover' FROM items WHERE collection='pgstac-test-collection'; $$,$$ SELECT '29'; $$, 'Test update_item function' ); select delete_item('pgstac-test-item-0003'); SELECT results_eq($$ SELECT count(*) FROM items WHERE collection='pgstac-test-collection'; $$,$$ SELECT 0::bigint; $$, 'Test delete_item function' ); ================================================ FILE: src/pgstac/tests/pgtap/004_search.sql ================================================ -- CREATE fixtures for testing search - as tests are run within a transaction, these will not persist \copy items_staging (content) FROM 'tests/testdata/items.ndjson' SET pgstac.context TO 'on'; SET pgstac."default_filter_lang" TO 'cql-json'; SELECT has_function('pgstac'::name, 'parse_dtrange', ARRAY['jsonb','timestamptz']); SELECT results_eq($$ SELECT parse_dtrange('["2020-01-01","2021-01-01"]'::jsonb) $$, $$ SELECT '["2020-01-01 00:00:00+00","2021-01-01 00:00:00+00")'::tstzrange $$, 'daterange passed as array range'); SELECT results_eq($$ SELECT parse_dtrange('"2020-01-01/2021-01-01"'::jsonb) $$, $$ SELECT '["2020-01-01 00:00:00+00","2021-01-01 00:00:00+00")'::tstzrange $$, 'date range passed as string range'); SELECT results_eq($$ SELECT parse_dtrange('"2020-01-01/.."'::jsonb) $$, $$ SELECT '["2020-01-01 00:00:00+00",infinity)'::tstzrange $$, 'date range passed as string range'); SELECT results_eq($$ SELECT parse_dtrange('"2020-01-01/"'::jsonb) $$, $$ SELECT '["2020-01-01 00:00:00+00",infinity)'::tstzrange $$, 'date range passed as string range'); SELECT results_eq($$ SELECT parse_dtrange('"../2020-01-01"'::jsonb) $$, $$ SELECT '[-infinity,"2020-01-01 00:00:00+00")'::tstzrange $$, 'date range passed as string range'); SELECT results_eq($$ SELECT parse_dtrange('"/2020-01-01"'::jsonb) $$, $$ SELECT '[-infinity,"2020-01-01 00:00:00+00")'::tstzrange $$, 'date range passed as string range'); SELECT has_function('pgstac'::name, 'bbox_geom', ARRAY['jsonb']); SELECT results_eq($$ SELECT bbox_geom('[0,1,2,3]') $$, $$ SELECT 'SRID=4326;POLYGON((0 1,0 3,2 3,2 1,0 1))'::geometry $$, '2d bbox'); SELECT results_eq($$ SELECT bbox_geom('[0,1,2,3,4,5]'::jsonb) $$, $$ SELECT '010F0000A0E610000006000000010300008001000000050000000000000000000000000000000000F03F00000000000000400000000000000000000000000000104000000000000000400000000000000840000000000000104000000000000000400000000000000840000000000000F03F00000000000000400000000000000000000000000000F03F0000000000000040010300008001000000050000000000000000000000000000000000F03F00000000000014400000000000000840000000000000F03F00000000000014400000000000000840000000000000104000000000000014400000000000000000000000000000104000000000000014400000000000000000000000000000F03F0000000000001440010300008001000000050000000000000000000000000000000000F03F00000000000000400000000000000000000000000000F03F00000000000014400000000000000000000000000000104000000000000014400000000000000000000000000000104000000000000000400000000000000000000000000000F03F0000000000000040010300008001000000050000000000000000000840000000000000F03F00000000000000400000000000000840000000000000104000000000000000400000000000000840000000000000104000000000000014400000000000000840000000000000F03F00000000000014400000000000000840000000000000F03F0000000000000040010300008001000000050000000000000000000000000000000000F03F00000000000000400000000000000840000000000000F03F00000000000000400000000000000840000000000000F03F00000000000014400000000000000000000000000000F03F00000000000014400000000000000000000000000000F03F000000000000004001030000800100000005000000000000000000000000000000000010400000000000000040000000000000000000000000000010400000000000001440000000000000084000000000000010400000000000001440000000000000084000000000000010400000000000000040000000000000000000000000000010400000000000000040'::geometry $$, '3d bbox'); SELECT has_function('pgstac'::name, 'sort_sqlorderby', ARRAY['jsonb','boolean']); SELECT results_eq($$ SELECT sort_sqlorderby('{"sortby":[{"field":"datetime","direction":"desc"},{"field":"eo:cloud_cover","direction":"asc"}]}'::jsonb); $$,$$ SELECT 'datetime DESC, to_int(content->''properties''->''eo:cloud_cover'') ASC, id DESC'; $$, 'Test creation of sort sql' ); SELECT results_eq($$ SELECT sort_sqlorderby('{"sortby":[{"field":"datetime","direction":"desc"},{"field":"eo:cloud_cover","direction":"asc"}]}'::jsonb, true); $$,$$ SELECT 'datetime ASC, to_int(content->''properties''->''eo:cloud_cover'') DESC, id ASC'; $$, 'Test creation of reverse sort sql' ); SELECT has_function('pgstac'::name, 'search', ARRAY['jsonb']); SELECT results_eq($$ SELECT search('{"collections": ["pgstac-test-collection"], "limit": 10, "sortby":[{"field":"id","direction":"asc"}], "token": "prev:pgstac-test-item-0011"}') $$,$$ SELECT search('{"collections": ["pgstac-test-collection"], "limit": 10, "sortby":[{"field":"id","direction":"asc"}]}') $$, 'Test prev token when reading first token_type=prev (https://github.com/stac-utils/pgstac/issues/140)' ); SELECT has_function('pgstac'::name, 'search_query', ARRAY['jsonb','boolean','jsonb']); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "intersects": { "type": "Polygon", "coordinates": [[ [-77.0824, 38.7886], [-77.0189, 38.7886], [-77.0189, 38.8351], [-77.0824, 38.8351], [-77.0824, 38.7886] ]] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ st_intersects(geometry, '0103000020E61000000100000005000000304CA60A464553C014D044D8F06443403E7958A8354153C014D044D8F06443403E7958A8354153C0DE718A8EE46A4340304CA60A464553C0DE718A8EE46A4340304CA60A464553C014D044D8F0644340') $r$,E' \n'); $$, 'Make sure that intersects returns valid query' ); -- CQL 2 Tests from examples at https://github.com/radiantearth/stac-api-spec/blob/f5da775080ff3ff46d454c2888b6e796ee956faf/fragments/filter/README.md SET pgstac."default_filter_lang" TO 'cql2-json'; SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter": { "op" : "and", "args": [ { "op": "=", "args": [ { "property": "id" }, "LC08_L1TP_060247_20180905_20180912_01_T1_L1TP" ] }, { "op": "=", "args" : [ { "property": "collection" }, "landsat8_l1tp" ] } ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ (id = 'LC08_L1TP_060247_20180905_20180912_01_T1_L1TP' AND collection = 'landsat8_l1tp') $r$,E' \n'); $$, 'Test Example 1' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "and", "args": [ { "op": "=", "args": [ { "property": "collection" }, "landsat8_l1tp" ] }, { "op": "<=", "args": [ { "property": "eo:cloud_cover" }, "10" ] }, { "op": ">=", "args": [ { "property": "datetime" }, {"timestamp": "2021-04-08T04:39:23Z"} ] }, { "op": "s_intersects", "args": [ { "property": "geometry" }, { "type": "Polygon", "coordinates": [ [ [43.5845, -79.5442], [43.6079, -79.4893], [43.5677, -79.4632], [43.6129, -79.3925], [43.6223, -79.3238], [43.6576, -79.3163], [43.7945, -79.1178], [43.8144, -79.1542], [43.8555, -79.1714], [43.7509, -79.6390], [43.5845, -79.5442] ] ] } ] } ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ (collection = 'landsat8_l1tp' AND to_int(content->'properties'->'eo:cloud_cover') <= to_int('"10"') AND datetime >= '2021-04-08 04:39:23+00'::timestamptz AND st_intersects(geometry, '0103000020E6100000010000000B000000894160E5D0CA4540ED9E3C2CD4E253C0849ECDAACFCD4540B37BF2B050DF53C038F8C264AAC8454076E09C11A5DD53C0F5DBD78173CE454085EB51B81ED953C08126C286A7CF4540789CA223B9D453C0C0EC9E3C2CD4454063EE5A423ED453C004560E2DB2E5454001DE02098AC753C063EE5A423EE84540C442AD69DEC953C02FDD240681ED454034A2B437F8CA53C08048BF7D1DE0454037894160E5E853C0894160E5D0CA4540ED9E3C2CD4E253C0'::geometry)) $r$,E' \n'); $$, 'Test Example 2' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "and", "args": [ { "op": ">", "args": [ { "property": "sentinel:data_coverage" }, "50" ] }, { "op": "<", "args": [ { "property": "eo:cloud_cover" }, 10 ] } ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ (to_text(content->'properties'->'sentinel:data_coverage') > to_text('"50"') AND to_int(content->'properties'->'eo:cloud_cover') < to_int('10')) $r$,E' \n'); $$, 'Test Example 3' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "or", "args": [ { "op": ">", "args": [ { "property": "sentinel:data_coverage" }, 50 ] }, { "op": "<", "args": [ { "property": "eo:cloud_cover" }, 10 ] } ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ (to_float(content->'properties'->'sentinel:data_coverage') > to_float('50') OR to_int(content->'properties'->'eo:cloud_cover') < to_int('10')) $r$,E' \n'); $$, 'Test Example 4' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "eq", "args": [ { "property": "prop1" }, { "property": "prop2" } ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ to_text(content->'properties'->'prop1') = to_text(content->'properties'->'prop2') $r$,E' \n'); $$, 'Test Example 5' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "t_intersects", "args": [ { "property": "datetime" }, { "interval": [ "2020-11-11T00:00:00Z", "2020-11-12T00:00:00Z"] } ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ (datetime <= '2020-11-12 00:00:00+00'::timestamptz AND end_datetime >= '2020-11-11 00:00:00+00'::timestamptz) $r$,E' \n'); $$, 'Test Example 6' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "s_intersects", "args": [ { "property": "geometry" } , { "type": "Polygon", "coordinates": [[ [-77.0824, 38.7886], [-77.0189, 38.7886], [-77.0189, 38.8351], [-77.0824, 38.8351], [-77.0824, 38.7886] ]] } ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ st_intersects(geometry, '0103000020E61000000100000005000000304CA60A464553C014D044D8F06443403E7958A8354153C014D044D8F06443403E7958A8354153C0DE718A8EE46A4340304CA60A464553C0DE718A8EE46A4340304CA60A464553C014D044D8F0644340'::geometry) $r$,E' \n'); $$, 'Test Example 7' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter": { "op": "or" , "args": [ { "op": "s_intersects", "args": [ { "property": "geometry" } , { "type": "Polygon", "coordinates": [[ [-77.0824, 38.7886], [-77.0189, 38.7886], [-77.0189, 38.8351], [-77.0824, 38.8351], [-77.0824, 38.7886] ]] } ] }, { "op": "s_intersects", "args": [ { "property": "geometry" } , { "type": "Polygon", "coordinates": [[ [-79.0935, 38.7886], [-79.0290, 38.7886], [-79.0290, 38.8351], [-79.0935, 38.8351], [-79.0935, 38.7886] ]] } ] } ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ (st_intersects(geometry, '0103000020E61000000100000005000000304CA60A464553C014D044D8F06443403E7958A8354153C014D044D8F06443403E7958A8354153C0DE718A8EE46A4340304CA60A464553C0DE718A8EE46A4340304CA60A464553C014D044D8F0644340'::geometry) OR st_intersects(geometry, '0103000020E61000000100000005000000448B6CE7FBC553C014D044D8F064434060E5D022DBC153C014D044D8F064434060E5D022DBC153C0DE718A8EE46A4340448B6CE7FBC553C0DE718A8EE46A4340448B6CE7FBC553C014D044D8F0644340'::geometry)) $r$,E' \n'); $$, 'Test Example 8' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "or", "args": [ { "op": ">=", "args": [ { "property": "sentinel:data_coverage" }, 50 ] }, { "op": ">=", "args": [ { "property": "landsat:coverage_percent" }, 50 ] }, { "op": "and", "args": [ { "op": "isNull", "args": { "property": "sentinel:data_coverage" } }, { "op": "isNull", "args": { "property": "landsat:coverage_percent" } } ] } ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ (to_float(content->'properties'->'sentinel:data_coverage') >= to_float('50') OR to_float(content->'properties'->'landsat:coverage_percent') >= to_float('50') OR (to_text(content->'properties'->'sentinel:data_coverage') IS NULL AND to_text(content->'properties'->'landsat:coverage_percent') IS NULL)) $r$,E' \n'); $$, 'Test Example 9' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "between", "args": [ { "property": "eo:cloud_cover" }, 0, 50 ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ to_int(content->'properties'->'eo:cloud_cover') BETWEEN to_int('0') AND to_int('50') $r$,E' \n'); $$, 'Test Example 10' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "like", "args": [ { "property": "mission" }, "sentinel%" ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ to_text(content->'properties'->'mission') LIKE to_text('"sentinel%"') $r$,E' \n'); $$, 'Test Example 11' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "eq", "args": [ {"upper": { "property": "mission" }}, {"upper": "sentinel"} ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ upper(to_text(content->'properties'->'mission')) = upper(to_text('"sentinel"')) $r$,E' \n'); $$, 'Test upper' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "eq", "args": [ {"lower": { "property": "mission" }}, {"lower": "sentinel"} ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ lower(to_text(content->'properties'->'mission')) = lower(to_text('"sentinel"')) $r$,E' \n'); $$, 'Test lower' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "eq", "args": [ {"op": "casei", "args":[{ "property": "mission" }]}, {"op": "casei", "args":["sentinel"]} ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ upper(to_text(content->'properties'->'mission')) = upper(to_text('"sentinel"')) $r$,E' \n'); $$, 'Test casei' ); SELECT results_eq($$ SELECT BTRIM(stac_search_to_where($q$ { "filter-lang": "cql2-json", "filter": { "op": "eq", "args": [ {"op": "accenti", "args":[{ "property": "mission" }]}, {"op": "accenti", "args":["sentinel"]} ] } } $q$),E' \n'); $$, $$ SELECT BTRIM($r$ unaccent(to_text(content->'properties'->'mission')) = unaccent(to_text('"sentinel"')) $r$,E' \n'); $$, 'Test accenti' ); /* template SELECT results_eq($$ $$,$$ $$, 'Test that ...' ); */ CREATE OR REPLACE FUNCTION pg_temp.isnull(j jsonb) RETURNS boolean AS $$ SELECT nullif(j, 'null'::jsonb) IS NULL; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION pg_temp.isnull(t text) RETURNS boolean AS $$ SELECT t IS NULL; $$ LANGUAGE SQL IMMUTABLE PARALLEL SAFE; CREATE OR REPLACE FUNCTION pg_temp.prev(j jsonb) RETURNS text AS $$ SELECT split_part(jsonb_path_query_first(j, '$.links[*] ? (@.rel == "prev") .href')->>0, 'token=', 2); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION pg_temp.next(j jsonb) RETURNS text AS $$ SELECT split_part(jsonb_path_query_first(j, '$.links[*] ? (@.rel == "next") .href')->>0, 'token=', 2); $$ LANGUAGE SQL IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION pg_temp.testpaging(testsortdir text, iddir text) RETURNS SETOF TEXT LANGUAGE plpgsql AS $$ DECLARE searchfilter jsonb; searchresult jsonb; offsetids text; searchresultids text; page int := 0; token text; BEGIN RAISE NOTICE 'Testing % %', testsortdir, iddir; -- Create collection with items that have a field with nulls and duplicate values DELETE FROM items WHERE collection = 'pgstac-test-collection2'; INSERT INTO collections (content) VALUES ('{"id":"pgstac-test-collection2"}'::jsonb) ON CONFLICT DO NOTHING; PERFORM check_partition('pgstac-test-collection2', '[2011-01-01,2012-01-01)', '[2011-01-01,2012-01-01)'); INSERT INTO items (id, collection, datetime, end_datetime, geometry, content) SELECT concat(id, '_2'), 'pgstac-test-collection2', datetime, end_datetime, geometry, content FROM items WHERE collection='pgstac-test-collection'; UPDATE items SET content = '{"properties":{"testsort":1}}'::jsonb WHERE collection = 'pgstac-test-collection2' AND id <= 'pgstac-test-item-0005_2'; UPDATE items SET content = '{"properties":{"testsort":2}}'::jsonb WHERE collection = 'pgstac-test-collection2' AND id > 'pgstac-test-item-0005_2' and id <= 'pgstac-test-item-0010_2'; UPDATE items SET content = '{"properties":{"testsort":3}}'::jsonb WHERE collection = 'pgstac-test-collection2' AND id > 'pgstac-test-item-0010' and id <= 'pgstac-test-item-0015_2'; RETURN NEXT results_eq( $q$ SELECT count(*) FROM items WHERE collection = 'pgstac-test-collection2'; $q$, $q$ SELECT 100::bigint; $q$, 'pgstac-test-collection2 has 100 items' ); searchfilter := '{"collections":["pgstac-test-collection2"],"fields":{"include":["id","properties.datetime","properties.testsort"]},"sortby":[{"field":"testsort","direction":null},{"field":"id","direction":null}]}'::jsonb; searchfilter := jsonb_set(searchfilter, '{sortby,0,direction}'::text[], to_jsonb(testsortdir)); searchfilter := jsonb_set(searchfilter, '{sortby,1,direction}'::text[], to_jsonb(iddir)); RAISE NOTICE 'SORTBY: %', searchfilter->>'sortby'; searchresult := search(searchfilter); RETURN NEXT ok(pg_temp.isnull(pg_temp.prev(searchresult)), 'first prev is null'); -- page up WHILE page <= 100 LOOP EXECUTE format($q$ WITH t AS ( SELECT id FROM items WHERE collection='pgstac-test-collection2' ORDER BY content->'properties'->>'testsort' %s, id %s OFFSET %L LIMIT 10 ) SELECT string_agg(id, ',') FROM t $q$, testsortdir, iddir, page ) INTO offsetids; EXECUTE format($q$ SELECT string_agg(q->>0, ',') FROM jsonb_path_query(%L, '$.features[*].id') as q; $q$, searchresult) INTO searchresultids; RAISE NOTICE 'O: %', offsetids; RAISE NOTICE 'S: %', searchresultids; RETURN NEXT results_eq( format($q$ SELECT id FROM items WHERE collection='pgstac-test-collection2' ORDER BY content->'properties'->>'testsort' %s, id %s OFFSET %L LIMIT 10 $q$, testsortdir, iddir, page ), format($q$ SELECT q->>0 FROM jsonb_path_query(%L, '$.features[*].id') as q; $q$, searchresult), format('Going up %s/%s page:%s results match using offset', testsortdir, iddir, page) ); IF pg_temp.isnull(pg_temp.next(searchresult)) THEN EXIT; END IF; searchfilter := searchfilter || jsonb_build_object('token', pg_temp.next(searchresult)); RAISE NOTICE 'SEARCHFILTER: %', searchfilter; searchresult := search(searchfilter); RAISE NOTICE 'SEARCHRESULT: %', searchresult; RAISE NOTICE 'PAGE:% TOKEN:% LINKS:%', page, searchfilter->>'token', searchresult->'links'; page := page + 10; END LOOP; RETURN NEXT ok(pg_temp.isnull(pg_temp.next(searchresult)), 'last next is null'); RETURN NEXT ok(page=90, 'last page going up is 90'); -- page down WHILE page >= 0 LOOP IF page < 10 THEN EXIT; END IF; page := page - 10; searchfilter := searchfilter || jsonb_build_object('token', pg_temp.prev(searchresult)); RAISE NOTICE 'SEARCHFILTER: %', searchfilter; searchresult := search(searchfilter); RAISE NOTICE 'SEARCHRESULT: %', searchresult; RAISE NOTICE 'PAGE:% TOKEN:% LINKS:%', page, searchfilter->>'token', searchresult->>'links'; EXECUTE format($q$ WITH t AS ( SELECT id FROM items WHERE collection='pgstac-test-collection2' ORDER BY content->'properties'->>'testsort' %s, id %s OFFSET %L LIMIT 10 ) SELECT string_agg(id, ',') FROM t $q$, testsortdir, iddir, page ) INTO offsetids; EXECUTE format($q$ SELECT string_agg(q->>0, ',') FROM jsonb_path_query(%L, '$.features[*].id') as q; $q$, searchresult) INTO searchresultids; RAISE NOTICE 'O: %', offsetids; RAISE NOTICE 'S: %', searchresultids; RETURN NEXT results_eq( format($q$ SELECT id FROM items WHERE collection='pgstac-test-collection2' ORDER BY content->'properties'->>'testsort' %s, id %s OFFSET %L LIMIT 10 $q$, testsortdir, iddir, page ), format($q$ SELECT q->>0 FROM jsonb_path_query(%L, '$.features[*].id') as q; $q$, searchresult), format('Going down %s/%s page:%s results match using offset', testsortdir, iddir, page) ); IF pg_temp.isnull(pg_temp.prev(searchresult)) THEN EXIT; END IF; END LOOP; RETURN NEXT ok(pg_temp.isnull(pg_temp.prev(searchresult)), 'last prev is null'); RETURN NEXT ok(page=0, 'last page going down is 0'); END; $$; SELECT * FROM pg_temp.testpaging('asc','asc'); SELECT * FROM pg_temp.testpaging('asc','desc'); SELECT * FROM pg_temp.testpaging('desc','desc'); SELECT * FROM pg_temp.testpaging('desc','asc'); \copy items_staging (content) FROM 'tests/testdata/items_duplicate_ids.ndjson' SELECT is( (SELECT jsonb_array_length(search('{"ids": ["pgstac-test-item-duplicated"]}')->'features')), '2', 'Make sure all matching items are returned when items with the same ID are in multiple collections, no collections specified. #192' ); SELECT is( (SELECT jsonb_array_length(search('{"ids": ["pgstac-test-item-duplicated"], "collections": ["pgstac-test-collection"]}')->'features')), '1', 'Make sure all matching items are returned when items with the same ID are in multiple collections, some collections specified. #192' ); SELECT is( (SELECT jsonb_array_length(search('{"ids": ["pgstac-test-item-duplicated"], "collections": ["pgstac-test-collection", "pgstac-test-collection2"]}')->'features')), '2', 'Make sure all matching items are returned when items with the same ID are in multiple collections, all collections specified. #192' ); ================================================ FILE: src/pgstac/tests/pgtap/004a_collectionsearch.sql ================================================ -- CREATE fixtures for testing search - as tests are run within a transaction, these will not persist SET pgstac.context TO 'on'; SET pgstac."default_filter_lang" TO 'cql2-json'; WITH t AS ( SELECT row_number() over () as id, x, y FROM generate_series(-180, 170, 10) as x, generate_series(-90, 80, 10) as y ), t1 AS ( SELECT concat('testcollection_', id) as id, x as minx, y as miny, x+10 as maxx, y+10 as maxy, '2000-01-01'::timestamptz + (concat(id, ' weeks'))::interval as sdt, '2000-01-01'::timestamptz + (concat(id, ' weeks'))::interval + ('2 months')::interval as edt FROM t ) SELECT create_collection(format($q$ { "id": "%s", "type": "Collection", "title": "My Test Collection.", "description": "Description of my test collection.", "extent": { "spatial": {"bbox": [[%s, %s, %s, %s]]}, "temporal": {"interval": [[%I, %I]]} }, "stac_extensions":[] } $q$, id, minx, miny, maxx, maxy, sdt, edt )::jsonb) FROM t1; SELECT has_function('pgstac'::name, 'collection_search', ARRAY['jsonb']); SELECT results_eq($$ select collection_search('{"ids":["testcollection_1","testcollection_2"],"limit":1, "sortby":[{"field":"id","direction":"asc"}]}'); $$,$$ SELECT '{"links": [{"rel": "next", "body": {"offset": 1}, "href": "./collections", "type": "application/json", "merge": true, "method": "GET"}], "numberMatched": 2, "numberReturned": 1, "collections": [{"id": "testcollection_1", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-180, -90, -170, -80]]}, "temporal": {"interval": [["2000-01-08 00:00:00+00", "2000-03-08 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}]}'::jsonb $$, 'Test search passing in collection ids' ); SELECT results_eq($$ select collection_search('{"ids":["testcollection_1","testcollection_2"],"limit":1, "sortby":[{"field":"id","direction":"desc"}]}'); $$,$$ SELECT '{"links": [{"rel": "next", "body": {"offset": 1}, "href": "./collections", "type": "application/json", "merge": true, "method": "GET"}], "numberMatched": 2, "numberReturned": 1, "collections": [{"id": "testcollection_2", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-170, -90, -160, -80]]}, "temporal": {"interval": [["2000-01-15 00:00:00+00", "2000-03-15 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}]}'::jsonb $$, 'Test search passing in collection ids with descending sort' ); SELECT results_eq($$ select collection_search('{"limit":1}') - '{collections}'::text[]; $$,$$ SELECT '{"links": [{"rel": "next", "body": {"offset": 1}, "href": "./collections", "type": "application/json", "merge": true, "method": "GET"}], "numberMatched": 650, "numberReturned": 1}'::jsonb $$, 'Test search limit 1 - next link' ); SELECT results_eq($$ select collection_search('{"limit":1, "offset":649}') - '{collections}'::text[]; $$,$$ SELECT '{"links": [{"rel": "prev", "body": {"offset": 648}, "href": "./collections", "type": "application/json", "merge": true, "method": "GET"}], "numberMatched": 650, "numberReturned": 1}'::jsonb $$, 'Test search limit 1, offset 649 - no next link' ); SELECT results_eq($$ select collection_search('{"limit": 700}') - '{collections}'::text[]; $$,$$ SELECT '{"links": [], "numberMatched": 650, "numberReturned": 650}'::jsonb $$, 'Test search limit 2 - no prev/next links' ); SET pgstac.base_url='https://test.com/'; SELECT results_eq($$ select collection_search('{"ids":["testcollection_1","testcollection_2"],"limit":1, "sortby":[{"field":"id","direction":"asc"}]}'); $$,$$ SELECT '{"links": [{"rel": "next", "body": {"offset": 1}, "href": "https://test.com/collections", "type": "application/json", "merge": true, "method": "GET"}], "numberMatched": 2, "numberReturned": 1, "collections": [{"id": "testcollection_1", "type": "Collection", "title": "My Test Collection.", "extent": {"spatial": {"bbox": [[-180, -90, -170, -80]]}, "temporal": {"interval": [["2000-01-08 00:00:00+00", "2000-03-08 00:00:00+00"]]}}, "description": "Description of my test collection.", "stac_extensions": []}]}'::jsonb $$, 'Test search passing in collection ids with base_url set' ); ================================================ FILE: src/pgstac/tests/pgtap/005_tileutils.sql ================================================ SELECT has_function('pgstac'::name, 'tileenvelope', ARRAY['int', 'int', 'int']); SELECT has_function('pgstac'::name, 'ftime', ARRAY[]::text[]); ================================================ FILE: src/pgstac/tests/pgtap/006_tilesearch.sql ================================================ SELECT has_function('pgstac'::name, 'geometrysearch', ARRAY['geometry','text','jsonb','int','int','interval','boolean','boolean']); SELECT has_function('pgstac'::name, 'geojsonsearch', ARRAY['jsonb','text','jsonb','int','int','interval','boolean','boolean']); SELECT has_function('pgstac'::name, 'xyzsearch', ARRAY['int','int','int','text','jsonb','int','int','interval','boolean','boolean']); ================================================ FILE: src/pgstac/tests/pgtap/9999_readonly.sql ================================================ SET transaction_read_only TO 'on'; SELECT results_eq( $$ SHOW transaction_read_only; $$, $$ SELECT 'on'; $$, 'Transaction set to read only' ); SELECT throws_ok( $$ SELECT search('{}'); $$, '25006' ); SET pgstac.readonly to 'true'; SELECT results_eq( $$ SELECT pgstac.readonly(); $$, $$ SELECT TRUE; $$, 'Readonly is set to true' ); SELECT lives_ok( $$ SELECT search('{}'); $$, 'Search works with readonly mode set to on in readonly mode.' ); SET pgstac.context TO 'on'; SELECT lives_ok( $$ SELECT search('{}'); $$, 'Search works with readonly mode set to on in readonly mode and the context extension enabled.' ); RESET pgstac.readonly; ================================================ FILE: src/pgstac/tests/pgtap/999_version.sql ================================================ ================================================ FILE: src/pgstac/tests/pgtap.sql ================================================ \unset ECHO \set QUIET 1 -- Turn off echo and keep things quiet. -- Format the output for nice TAP. \pset format unaligned \pset tuples_only true \pset pager off \timing off -- Revert all changes on failure. \set ON_ERROR_STOP true -- Load the TAP functions. BEGIN; CREATE EXTENSION IF NOT EXISTS pgtap; SET SEARCH_PATH TO pgstac, pgtap, public; -- Plan the tests. SELECT plan(229); --SELECT * FROM no_plan(); -- Run the tests. -- Core \i tests/pgtap/001_core.sql \i tests/pgtap/001a_jsonutils.sql \i tests/pgtap/001b_cursorutils.sql \i tests/pgtap/001s_stacutils.sql \i tests/pgtap/002_collections.sql \i tests/pgtap/002a_queryables.sql \i tests/pgtap/003_items.sql \i tests/pgtap/004_search.sql \i tests/pgtap/004a_collectionsearch.sql \i tests/pgtap/005_tileutils.sql \i tests/pgtap/006_tilesearch.sql \i tests/pgtap/999_version.sql -- Finish the tests and clean up. SELECT * FROM finish(); ROLLBACK; ================================================ FILE: src/pgstac/tests/testdata/collections.json ================================================ { "id": "pgstac-test-collection", "stac_version": "1.0.0-beta.2", "description": "The National Agriculture Imagery Program (NAIP) acquires aerial imagery\nduring the agricultural growing seasons in the continental U.S.\n\nNAIP projects are contracted each year based upon available funding and the\nFSA imagery acquisition cycle. Beginning in 2003, NAIP was acquired on\na 5-year cycle. 2008 was a transition year, and a three-year cycle began\nin 2009.\n\nNAIP imagery is acquired at a one-meter ground sample distance (GSD) with a\nhorizontal accuracy that matches within six meters of photo-identifiable\nground control points, which are used during image inspection.\n\nOlder images were collected using 3 bands (Red, Green, and Blue: RGB), but\nnewer imagery is usually collected with an additional near-infrared band\n(RGBN).", "links": [ { "rel": "root", "href": "/collection.json", "type": "application/json" }, { "rel": "self", "href": "/collection.json", "type": "application/json" } ], "stac_extensions": [], "title": "NAIP: National Agriculture Imagery Program", "extent": { "spatial": { "bbox": [ [ -124.784, 24.744, -66.951, 49.346 ] ] }, "temporal": { "interval": [ [ "2011-01-01T00:00:00Z", "2019-01-01T00:00:00Z" ] ] } }, "license": "PDDL-1.0", "providers": [ { "name": "USDA Farm Service Agency", "roles": [ "producer", "licensor" ], "url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/" } ], "item_assets": { "image": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "RGBIR COG tile", "eo:bands": [ { "name": "Red", "common_name": "red" }, { "name": "Green", "common_name": "green" }, { "name": "Blue", "common_name": "blue" }, { "name": "NIR", "common_name": "nir", "description": "near-infrared" } ] }, "metadata": { "type": "text/plain", "roles": [ "metadata" ], "title": "FGDC Metdata" }, "thumbnail": { "type": "image/jpeg", "roles": [ "thumbnail" ], "title": "Thumbnail" } } } ================================================ FILE: src/pgstac/tests/testdata/collections.ndjson ================================================ {"id":"pgstac-test-collection","stac_version":"1.0.0-beta.2","description":"The National Agriculture Imagery Program (NAIP) acquires aerial imagery\\nduring the agricultural growing seasons in the continental U.S.\\n\\nNAIP projects are contracted each year based upon available funding and the\\nFSA imagery acquisition cycle. Beginning in 2003, NAIP was acquired on\\na 5-year cycle. 2008 was a transition year, and a three-year cycle began\\nin 2009.\\n\\nNAIP imagery is acquired at a one-meter ground sample distance (GSD) with a\\nhorizontal accuracy that matches within six meters of photo-identifiable\\nground control points, which are used during image inspection.\\n\\nOlder images were collected using 3 bands (Red, Green, and Blue: RGB), but\\nnewer imagery is usually collected with an additional near-infrared band\\n(RGBN).","links":[{"rel":"root","href":"/collection.json","type":"application/json"},{"rel":"self","href":"/collection.json","type":"application/json"}],"stac_extensions":[],"title":"NAIP: National Agriculture Imagery Program","extent":{"spatial":{"bbox":[[-124.784,24.744,-66.951,49.346]]},"temporal":{"interval":[["2011-01-01T00:00:00Z","2019-01-01T00:00:00Z"]]}},"license":"PDDL-1.0","providers":[{"name":"USDA Farm Service Agency","roles":["producer","licensor"],"url":"https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/"}],"item_assets":{"image":{"type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"title":"RGBIR COG tile","eo:bands":[{"name":"Red","common_name":"red"},{"name":"Green","common_name":"green"},{"name":"Blue","common_name":"blue"},{"name":"NIR","common_name":"nir","description":"near-infrared"}]},"metadata":{"type":"text/plain","roles":["metadata"],"title":"FGDC Metdata"},"thumbnail":{"type":"image/jpeg","roles":["thumbnail"],"title":"Thumbnail"}}} {"id":"pgstac-test-collection2","stac_version":"1.0.0-beta.2","description":"The National Agriculture Imagery Program (NAIP) acquires aerial imagery\\nduring the agricultural growing seasons in the continental U.S.\\n\\nNAIP projects are contracted each year based upon available funding and the\\nFSA imagery acquisition cycle. Beginning in 2003, NAIP was acquired on\\na 5-year cycle. 2008 was a transition year, and a three-year cycle began\\nin 2009.\\n\\nNAIP imagery is acquired at a one-meter ground sample distance (GSD) with a\\nhorizontal accuracy that matches within six meters of photo-identifiable\\nground control points, which are used during image inspection.\\n\\nOlder images were collected using 3 bands (Red, Green, and Blue: RGB), but\\nnewer imagery is usually collected with an additional near-infrared band\\n(RGBN).","links":[{"rel":"root","href":"/collection.json","type":"application/json"},{"rel":"self","href":"/collection.json","type":"application/json"}],"stac_extensions":[],"title":"NAIP: National Agriculture Imagery Program","extent":{"spatial":{"bbox":[[-124.784,24.744,-66.951,49.346]]},"temporal":{"interval":[["2011-01-01T00:00:00Z","2019-01-01T00:00:00Z"]]}},"license":"PDDL-1.0","providers":[{"name":"USDA Farm Service Agency","roles":["producer","licensor"],"url":"https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/"}],"item_assets":{"image":{"type":"image/tiff; application=geotiff; profile=cloud-optimized","roles":["data"],"title":"RGBIR COG tile","eo:bands":[{"name":"Red","common_name":"red"},{"name":"Green","common_name":"green"},{"name":"Blue","common_name":"blue"},{"name":"NIR","common_name":"nir","description":"near-infrared"}]},"metadata":{"type":"text/plain","roles":["metadata"],"title":"FGDC Metdata"},"thumbnail":{"type":"image/jpeg","roles":["thumbnail"],"title":"Thumbnail"}}} ================================================ FILE: src/pgstac/tests/testdata/items.ndjson ================================================ {"id": "pgstac-test-item-0003", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0004", "bbox": [-87.190775, 30.934712, -87.121772, 31.002794], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_ne_16_1_20110824.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008707_ne_16_1_20110824.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_ne_16_1_20110824.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.121772, 30.934795], [-87.121858, 31.002794], [-87.190775, 31.002711], [-87.190639, 30.934712], [-87.121772, 30.934795]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [481788, 3422382, 488367, 3429918], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7536, 6579], "eo:cloud_cover": 23, "proj:transform": [1, 0, 481788, 0, -1, 3429918, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0005", "bbox": [-87.128238, 30.934744, -87.059311, 31.002757], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_nw_16_1_20110824.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008708_nw_16_1_20110824.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_nw_16_1_20110824.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.059311, 30.934794], [-87.059353, 31.002757], [-87.128238, 31.002707], [-87.128147, 30.934744], [-87.059311, 30.934794]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [487758, 3422377, 494334, 3429909], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7532, 6576], "eo:cloud_cover": 3, "proj:transform": [1, 0, 487758, 0, -1, 3429909, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0006", "bbox": [-87.253322, 30.934677, -87.184223, 31.00282], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_nw_16_1_20110824.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008707_nw_16_1_20110824.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_nw_16_1_20110824.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.184223, 30.934794], [-87.184354, 31.00282], [-87.253322, 31.002703], [-87.253142, 30.934677], [-87.184223, 30.934794]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [475817, 3422390, 482401, 3429929], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7539, 6584], "eo:cloud_cover": 100, "proj:transform": [1, 0, 475817, 0, -1, 3429929, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0007", "bbox": [-87.440935, 30.622081, -87.371617, 30.690415], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_se_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_se_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_se_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.371617, 30.622297], [-87.371878, 30.690415], [-87.440935, 30.690199], [-87.440626, 30.622081], [-87.371617, 30.622297]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [457770, 3387803, 464384, 3395352], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7549, 6614], "eo:cloud_cover": 59, "proj:transform": [1, 0, 457770, 0, -1, 3395352, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0008", "bbox": [-87.503478, 30.622053, -87.434074, 30.690447], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_sw_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_sw_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_sw_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.434074, 30.622302], [-87.434379, 30.690447], [-87.503478, 30.690198], [-87.503124, 30.622053], [-87.434074, 30.622302]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [451780, 3387825, 458398, 3395377], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7552, 6618], "eo:cloud_cover": 64, "proj:transform": [1, 0, 451780, 0, -1, 3395377, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0009", "bbox": [-87.503478, 30.68455, -87.434072, 30.752944], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_nw_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_nw_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_nw_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.434072, 30.684799], [-87.434377, 30.752944], [-87.503478, 30.752694], [-87.503125, 30.68455], [-87.434072, 30.684799]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [451811, 3394751, 458425, 3402303], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7552, 6614], "eo:cloud_cover": 61, "proj:transform": [1, 0, 451811, 0, -1, 3402303, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0010", "bbox": [-87.440937, 30.684579, -87.371606, 30.752912], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_ne_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_ne_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_ne_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.371606, 30.684794], [-87.371867, 30.752912], [-87.440937, 30.752696], [-87.440628, 30.684579], [-87.371606, 30.684794]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [457797, 3394729, 464408, 3402278], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7549, 6611], "eo:cloud_cover": 31, "proj:transform": [1, 0, 457797, 0, -1, 3402278, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0011", "bbox": [-87.315859, 30.934648, -87.246684, 31.002851], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_ne_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008706_ne_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_ne_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.246684, 30.934798], [-87.246859, 31.002851], [-87.315859, 31.0027], [-87.315635, 30.934648], [-87.246684, 30.934798]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [469847, 3422402, 476434, 3429944], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7542, 6587], "eo:cloud_cover": 41, "proj:transform": [1, 0, 469847, 0, -1, 3429944, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0012", "bbox": [-87.440942, 30.93458, -87.371596, 31.002921], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_ne_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008705_ne_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_ne_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.371596, 30.934797], [-87.37186, 31.002921], [-87.440942, 31.002704], [-87.440629, 30.93458], [-87.371596, 30.934797]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [457906, 3422435, 464501, 3429985], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7550, 6595], "eo:cloud_cover": 4, "proj:transform": [1, 0, 457906, 0, -1, 3429985, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0013", "bbox": [-87.378406, 30.934615, -87.309145, 31.002887], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_nw_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008706_nw_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_nw_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.309145, 30.934799], [-87.309365, 31.002887], [-87.378406, 31.002704], [-87.378137, 30.934615], [-87.309145, 30.934799]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [463876, 3422417, 470467, 3429963], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7546, 6591], "eo:cloud_cover": 2, "proj:transform": [1, 0, 463876, 0, -1, 3429963, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0014", "bbox": [-87.628547, 30.496984, -87.558991, 30.565507], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558991, 30.497299], [-87.559382, 30.565507], [-87.628547, 30.565192], [-87.628108, 30.496984], [-87.558991, 30.497299]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439724, 3374025, 446357, 3381584], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6633], "eo:cloud_cover": 17, "proj:transform": [1, 0, 439724, 0, -1, 3381584, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0015", "bbox": [-86.940689, 30.934744, -86.871762, 31.002757], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008601_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.871853, 30.934744], [-86.871762, 31.002707], [-86.940647, 31.002757], [-86.940689, 30.934794], [-86.871853, 30.934744]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [505666, 3422377, 512242, 3429909], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7532, 6576], "eo:cloud_cover": 54, "proj:transform": [1, 0, 505666, 0, -1, 3429909, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0016", "bbox": [-87.003143, 30.934773, -86.93431, 31.002726], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008601_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.934356, 30.934773], [-86.93431, 31.002709], [-87.003143, 31.002726], [-87.00314, 30.934789], [-86.934356, 30.934773]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [499700, 3422375, 506271, 3429904], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7529, 6571], "eo:cloud_cover": 13, "proj:transform": [1, 0, 499700, 0, -1, 3429904, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0017", "bbox": [-86.815777, 30.934677, -86.746678, 31.00282], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008602_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.746858, 30.934677], [-86.746678, 31.002703], [-86.815646, 31.00282], [-86.815777, 30.934794], [-86.746858, 30.934677]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [517599, 3422390, 524183, 3429929], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7539, 6584], "eo:cloud_cover": 59, "proj:transform": [1, 0, 517599, 0, -1, 3429929, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0018", "bbox": [-86.878228, 30.934712, -86.809225, 31.002794], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008602_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.809361, 30.934712], [-86.809225, 31.002711], [-86.878142, 31.002794], [-86.878228, 30.934795], [-86.809361, 30.934712]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [511633, 3422382, 518212, 3429918], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7536, 6579], "eo:cloud_cover": 29, "proj:transform": [1, 0, 511633, 0, -1, 3429918, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0019", "bbox": [-87.566035, 30.934518, -87.496517, 31.002979], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008704_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496517, 30.934802], [-87.49687, 31.002979], [-87.566035, 31.002694], [-87.565633, 30.934518], [-87.496517, 30.934802]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445964, 3422482, 452567, 3430038], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6603], "eo:cloud_cover": 52, "proj:transform": [1, 0, 445964, 0, -1, 3430038, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0020", "bbox": [-87.62857, 30.934491, -87.558977, 31.003012], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008704_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558977, 30.934808], [-87.559374, 31.003012], [-87.62857, 31.002694], [-87.628123, 30.934491], [-87.558977, 30.934808]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439994, 3422511, 446600, 3430070], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6606], "eo:cloud_cover": 39, "proj:transform": [1, 0, 439994, 0, -1, 3430070, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0021", "bbox": [-87.628569, 30.871988, -87.55898, 30.940509], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008704_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.55898, 30.872305], [-87.559377, 30.940509], [-87.628569, 30.940191], [-87.628123, 30.871988], [-87.55898, 30.872305]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439955, 3415584, 446565, 3423143], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6610], "eo:cloud_cover": 29, "proj:transform": [1, 0, 439955, 0, -1, 3423143, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0022", "bbox": [-87.503488, 30.93455, -87.434056, 31.002952], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008705_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.434056, 30.934801], [-87.434365, 31.002952], [-87.503488, 31.002701], [-87.503131, 30.93455], [-87.434056, 30.934801]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [451935, 3422457, 458534, 3430010], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7553, 6599], "eo:cloud_cover": 84, "proj:transform": [1, 0, 451935, 0, -1, 3430010, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0023", "bbox": [-87.06569, 30.934773, -86.996857, 31.002726], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008708_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.99686, 30.934789], [-86.996857, 31.002726], [-87.06569, 31.002709], [-87.065644, 30.934773], [-86.99686, 30.934789]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [493729, 3422375, 500300, 3429904], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7529, 6571], "eo:cloud_cover": 56, "proj:transform": [1, 0, 493729, 0, -1, 3429904, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0024", "bbox": [-87.941273, 30.809325, -87.871271, 30.878173], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871271, 30.80981], [-87.871889, 30.878173], [-87.941273, 30.877687], [-87.940605, 30.809325], [-87.871271, 30.80981]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [410024, 3408849, 416657, 3416426], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6633], "eo:cloud_cover": 58, "proj:transform": [1, 0, 410024, 0, -1, 3416426, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0025", "bbox": [-88.003818, 30.809299, -87.933721, 30.878206], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.933721, 30.809817], [-87.934384, 30.878206], [-88.003818, 30.877686], [-88.003106, 30.809299], [-87.933721, 30.809817]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [404045, 3408898, 410683, 3416478], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6638], "eo:cloud_cover": 65, "proj:transform": [1, 0, 404045, 0, -1, 3416478, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0026", "bbox": [-87.941269, 30.746832, -87.871272, 30.815671], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871272, 30.747317], [-87.871889, 30.815671], [-87.941269, 30.815185], [-87.940603, 30.746832], [-87.871272, 30.747317]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409966, 3401923, 416603, 3409499], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6637], "eo:cloud_cover": 52, "proj:transform": [1, 0, 409966, 0, -1, 3409499, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0027", "bbox": [-88.003816, 30.746797, -87.933724, 30.815704], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.933724, 30.747315], [-87.934385, 30.815704], [-88.003816, 30.815185], [-88.003106, 30.746797], [-87.933724, 30.747315]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [403983, 3401971, 410625, 3409551], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6642], "eo:cloud_cover": 43, "proj:transform": [1, 0, 403983, 0, -1, 3409551, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0028", "bbox": [-87.878737, 30.809358, -87.80881, 30.878137], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.80881, 30.809809], [-87.809384, 30.878137], [-87.878737, 30.877684], [-87.878114, 30.809358], [-87.80881, 30.809809]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [416002, 3408804, 422632, 3416377], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6630], "eo:cloud_cover": 46, "proj:transform": [1, 0, 416002, 0, -1, 3416377, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0029", "bbox": [-87.878732, 30.746864, -87.80881, 30.815634], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.80881, 30.747315], [-87.809382, 30.815634], [-87.878732, 30.815182], [-87.878111, 30.746864], [-87.80881, 30.747315]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415948, 3401878, 422582, 3409450], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7572, 6634], "eo:cloud_cover": 42, "proj:transform": [1, 0, 415948, 0, -1, 3409450, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0030", "bbox": [-87.691106, 30.809455, -87.621436, 30.878045], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621436, 30.809805], [-87.621876, 30.878045], [-87.691106, 30.877694], [-87.690617, 30.809455], [-87.621436, 30.809805]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433938, 3408689, 440556, 3416252], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6618], "eo:cloud_cover": 16, "proj:transform": [1, 0, 433938, 0, -1, 3416252, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0031", "bbox": [-87.691108, 30.74696, -87.621442, 30.815542], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621442, 30.74731], [-87.62188, 30.815542], [-87.691108, 30.815191], [-87.69062, 30.74696], [-87.621442, 30.74731]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433895, 3401763, 440517, 3409325], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6622], "eo:cloud_cover": 16, "proj:transform": [1, 0, 433895, 0, -1, 3409325, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0032", "bbox": [-87.628569, 30.809484, -87.558973, 30.878015], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008712_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558973, 30.809801], [-87.559369, 30.878015], [-87.628569, 30.877697], [-87.628124, 30.809484], [-87.558973, 30.809801]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439916, 3408657, 446531, 3416217], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7560, 6615], "eo:cloud_cover": 7, "proj:transform": [1, 0, 439916, 0, -1, 3416217, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0033", "bbox": [-87.566019, 30.747015, -87.496524, 30.815477], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008712_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496524, 30.747298], [-87.496874, 30.815477], [-87.566019, 30.815193], [-87.56562, 30.747015], [-87.496524, 30.747298]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445860, 3401702, 452474, 3409258], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6614], "eo:cloud_cover": 49, "proj:transform": [1, 0, 445860, 0, -1, 3409258, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0034", "bbox": [-87.628559, 30.746989, -87.558978, 30.815511], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008712_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558978, 30.747305], [-87.559372, 30.815511], [-87.628559, 30.815194], [-87.628115, 30.746989], [-87.558978, 30.747305]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439878, 3401731, 446496, 3409290], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6618], "eo:cloud_cover": 3, "proj:transform": [1, 0, 439878, 0, -1, 3409290, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0035", "bbox": [-87.941266, 30.68433, -87.871274, 30.753168], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871274, 30.684813], [-87.871889, 30.753168], [-87.941266, 30.752684], [-87.940602, 30.68433], [-87.871274, 30.684813]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409908, 3394996, 416549, 3402572], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6641], "eo:cloud_cover": 33, "proj:transform": [1, 0, 409908, 0, -1, 3402572, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0036", "bbox": [-88.003814, 30.684295, -87.933727, 30.753202], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.933727, 30.684813], [-87.934386, 30.753202], [-88.003814, 30.752684], [-88.003106, 30.684295], [-87.933727, 30.684813]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [403921, 3395044, 410567, 3402624], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6646], "eo:cloud_cover": 12, "proj:transform": [1, 0, 403921, 0, -1, 3402624, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0037", "bbox": [-87.941265, 30.621827, -87.871277, 30.690665], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871277, 30.62231], [-87.871891, 30.690665], [-87.941265, 30.690181], [-87.940603, 30.621827], [-87.871277, 30.62231]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409850, 3388069, 416495, 3395645], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6645], "eo:cloud_cover": 54, "proj:transform": [1, 0, 409850, 0, -1, 3395645, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0038", "bbox": [-88.003804, 30.621802, -87.933732, 30.6907], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.933732, 30.622318], [-87.934389, 30.6907], [-88.003804, 30.690182], [-88.003098, 30.621802], [-87.933732, 30.622318]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [403860, 3388118, 410509, 3395697], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7579, 6649], "eo:cloud_cover": 77, "proj:transform": [1, 0, 403860, 0, -1, 3395697, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0039", "bbox": [-87.878728, 30.684361, -87.808821, 30.75314], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808821, 30.684811], [-87.809392, 30.75314], [-87.878728, 30.752689], [-87.878109, 30.684361], [-87.808821, 30.684811]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415894, 3394951, 422531, 3402524], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6637], "eo:cloud_cover": 12, "proj:transform": [1, 0, 415894, 0, -1, 3402524, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0040", "bbox": [-87.878725, 30.621858, -87.808823, 30.690637], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808823, 30.622307], [-87.809392, 30.690637], [-87.878725, 30.690186], [-87.878107, 30.621858], [-87.808823, 30.622307]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415840, 3388024, 422481, 3395597], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6641], "eo:cloud_cover": 10, "proj:transform": [1, 0, 415840, 0, -1, 3395597, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0041", "bbox": [-87.6911, 30.684456, -87.621438, 30.753047], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621438, 30.684805], [-87.621875, 30.753047], [-87.6911, 30.752696], [-87.690613, 30.684456], [-87.621438, 30.684805]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433853, 3394836, 440479, 3402399], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6626], "eo:cloud_cover": 3, "proj:transform": [1, 0, 433853, 0, -1, 3402399, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0042", "bbox": [-87.691103, 30.62196, -87.621445, 30.690542], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621445, 30.622309], [-87.621882, 30.690542], [-87.691103, 30.690192], [-87.690618, 30.62196], [-87.621445, 30.622309]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433810, 3387910, 440440, 3395472], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6630], "eo:cloud_cover": 99, "proj:transform": [1, 0, 433810, 0, -1, 3395472, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0043", "bbox": [-87.566019, 30.684519, -87.496527, 30.752981], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496527, 30.684801], [-87.496877, 30.752981], [-87.566019, 30.752698], [-87.565621, 30.684519], [-87.496527, 30.684801]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445825, 3394776, 452443, 3402332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6618], "eo:cloud_cover": 80, "proj:transform": [1, 0, 445825, 0, -1, 3402332, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0044", "bbox": [-87.62856, 30.684484, -87.558983, 30.753015], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558983, 30.6848], [-87.559376, 30.753015], [-87.62856, 30.752699], [-87.628117, 30.684484], [-87.558983, 30.6848]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439839, 3394804, 446461, 3402364], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7560, 6622], "eo:cloud_cover": 30, "proj:transform": [1, 0, 439839, 0, -1, 3402364, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0045", "bbox": [-87.56602, 30.622022, -87.496532, 30.690476], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496532, 30.622304], [-87.49688, 30.690476], [-87.56602, 30.690193], [-87.565623, 30.622022], [-87.496532, 30.622304]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445790, 3387850, 452412, 3395405], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7555, 6622], "eo:cloud_cover": 82, "proj:transform": [1, 0, 445790, 0, -1, 3395405, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0046", "bbox": [-87.628562, 30.621988, -87.558988, 30.69051], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558988, 30.622303], [-87.559381, 30.69051], [-87.628562, 30.690194], [-87.62812, 30.621988], [-87.558988, 30.622303]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439800, 3387878, 446426, 3395437], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6626], "eo:cloud_cover": 16, "proj:transform": [1, 0, 439800, 0, -1, 3395437, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0047", "bbox": [-87.941265, 30.559332, -87.871282, 30.628171], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008725_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871282, 30.559814], [-87.871893, 30.628171], [-87.941265, 30.627687], [-87.940604, 30.559332], [-87.871282, 30.559814]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409792, 3381143, 416441, 3388719], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6649], "eo:cloud_cover": 43, "proj:transform": [1, 0, 409792, 0, -1, 3388719, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0048", "bbox": [-87.941255, 30.496828, -87.871287, 30.565666], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008725_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871287, 30.497309], [-87.871897, 30.565666], [-87.941255, 30.565183], [-87.940596, 30.496828], [-87.871287, 30.497309]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409735, 3374216, 416387, 3381792], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6652], "eo:cloud_cover": 4, "proj:transform": [1, 0, 409735, 0, -1, 3381792, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0049", "bbox": [-87.878723, 30.559362, -87.808825, 30.628132], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808825, 30.559811], [-87.809393, 30.628132], [-87.878723, 30.627682], [-87.878107, 30.559362], [-87.808825, 30.559811]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415786, 3381098, 422431, 3388670], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7572, 6645], "eo:cloud_cover": 9, "proj:transform": [1, 0, 415786, 0, -1, 3388670, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0050", "bbox": [-87.878723, 30.496867, -87.808828, 30.565637], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808828, 30.497315], [-87.809395, 30.565637], [-87.878723, 30.565187], [-87.878108, 30.496867], [-87.808828, 30.497315]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415732, 3374172, 422381, 3381744], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7572, 6649], "eo:cloud_cover": 84, "proj:transform": [1, 0, 415732, 0, -1, 3381744, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0051", "bbox": [-87.691097, 30.559454, -87.621453, 30.628045], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621453, 30.559803], [-87.621889, 30.628045], [-87.691097, 30.627696], [-87.690613, 30.559454], [-87.621453, 30.559803]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433768, 3380983, 440401, 3388546], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6633], "eo:cloud_cover": 22, "proj:transform": [1, 0, 433768, 0, -1, 3388546, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0052", "bbox": [-87.691091, 30.496957, -87.621451, 30.565539], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621451, 30.497305], [-87.621886, 30.565539], [-87.691091, 30.565191], [-87.690608, 30.496957], [-87.621451, 30.497305]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433726, 3374057, 440363, 3381619], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6637], "eo:cloud_cover": 12, "proj:transform": [1, 0, 433726, 0, -1, 3381619, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0053", "bbox": [-87.56601, 30.559516, -87.496536, 30.627979], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496536, 30.559797], [-87.496884, 30.627979], [-87.56601, 30.627696], [-87.565614, 30.559516], [-87.496536, 30.559797]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445756, 3380923, 452381, 3388479], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6625], "eo:cloud_cover": 24, "proj:transform": [1, 0, 445756, 0, -1, 3388479, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0054", "bbox": [-87.628554, 30.559491, -87.558995, 30.628014], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558995, 30.559806], [-87.559386, 30.628014], [-87.628554, 30.627698], [-87.628114, 30.559491], [-87.558995, 30.559806]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439762, 3380952, 446391, 3388511], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6629], "eo:cloud_cover": 5, "proj:transform": [1, 0, 439762, 0, -1, 3388511, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0055", "bbox": [-87.566012, 30.497018, -87.496531, 30.565481], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496531, 30.497299], [-87.496878, 30.565481], [-87.566012, 30.565199], [-87.565617, 30.497018], [-87.496531, 30.497299]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445721, 3373997, 452351, 3381553], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6630], "eo:cloud_cover": 79, "proj:transform": [1, 0, 445721, 0, -1, 3381553, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0056", "bbox": [-87.941283, 30.934327, -87.871262, 31.003175], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871262, 30.934813], [-87.871883, 31.003175], [-87.941283, 31.002688], [-87.940613, 30.934327], [-87.871262, 30.934813]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [410140, 3422703, 416766, 3430280], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6626], "eo:cloud_cover": 62, "proj:transform": [1, 0, 410140, 0, -1, 3430280, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0057", "bbox": [-88.003827, 30.9343, -87.93372, 31.003207], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.93372, 30.93482], [-87.934386, 31.003207], [-88.003827, 31.002686], [-88.003111, 30.9343], [-87.93372, 30.93482]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [404169, 3422752, 410799, 3430332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6630], "eo:cloud_cover": 7, "proj:transform": [1, 0, 404169, 0, -1, 3430332, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0058", "bbox": [-87.941277, 30.871827, -87.871261, 30.940674], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871261, 30.872312], [-87.87188, 30.940674], [-87.941277, 30.940187], [-87.940609, 30.871827], [-87.871261, 30.872312]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [410082, 3415776, 416712, 3423353], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6630], "eo:cloud_cover": 53, "proj:transform": [1, 0, 410082, 0, -1, 3423353, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0059", "bbox": [-88.003822, 30.8718, -87.93372, 30.940707], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.93372, 30.872319], [-87.934384, 30.940707], [-88.003822, 30.940186], [-88.003108, 30.8718], [-87.93372, 30.872319]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [404107, 3415825, 410741, 3423405], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6634], "eo:cloud_cover": 57, "proj:transform": [1, 0, 404107, 0, -1, 3423405, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0066", "bbox": [-86.441023, 30.934491, -86.37143, 31.003012], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008605_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.371877, 30.934491], [-86.37143, 31.002694], [-86.440626, 31.003012], [-86.441023, 30.934808], [-86.371877, 30.934491]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [553400, 3422511, 560006, 3430070], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6606], "eo:cloud_cover": 86, "proj:transform": [1, 0, 553400, 0, -1, 3430070, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0060", "bbox": [-87.878739, 30.934361, -87.808804, 31.00314], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808804, 30.934813], [-87.80938, 31.00314], [-87.878739, 31.002686], [-87.878114, 30.934361], [-87.808804, 30.934813]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [416111, 3422658, 422733, 3430231], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6622], "eo:cloud_cover": 70, "proj:transform": [1, 0, 416111, 0, -1, 3430231, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0061", "bbox": [-87.878743, 30.87186, -87.808801, 30.940638], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808801, 30.872311], [-87.809376, 30.940638], [-87.878743, 30.940186], [-87.878119, 30.87186], [-87.808801, 30.872311]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [416056, 3415731, 422683, 3423304], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6627], "eo:cloud_cover": 94, "proj:transform": [1, 0, 416056, 0, -1, 3423304, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0062", "bbox": [-87.691116, 30.934452, -87.621426, 31.003042], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621426, 30.934803], [-87.621868, 31.003042], [-87.691116, 31.00269], [-87.690624, 30.934452], [-87.621426, 30.934803]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [434023, 3422542, 440634, 3430105], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6611], "eo:cloud_cover": 85, "proj:transform": [1, 0, 434023, 0, -1, 3430105, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0063", "bbox": [-87.691116, 30.871958, -87.621431, 30.940539], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621431, 30.872309], [-87.621872, 30.940539], [-87.691116, 30.940188], [-87.690626, 30.871958], [-87.621431, 30.872309]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433980, 3415616, 440595, 3423178], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6615], "eo:cloud_cover": 2, "proj:transform": [1, 0, 433980, 0, -1, 3423178, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0064", "bbox": [-86.565944, 30.93455, -86.496512, 31.002952], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008604_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.496869, 30.93455], [-86.496512, 31.002701], [-86.565635, 31.002952], [-86.565944, 30.934801], [-86.496869, 30.93455]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [541466, 3422457, 548065, 3430010], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7553, 6599], "eo:cloud_cover": 40, "proj:transform": [1, 0, 541466, 0, -1, 3430010, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0065", "bbox": [-86.628404, 30.93458, -86.559058, 31.002921], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008604_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.559371, 30.93458], [-86.559058, 31.002704], [-86.62814, 31.002921], [-86.628404, 30.934797], [-86.559371, 30.93458]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [535499, 3422435, 542094, 3429985], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7550, 6595], "eo:cloud_cover": 27, "proj:transform": [1, 0, 535499, 0, -1, 3429985, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0067", "bbox": [-86.503483, 30.934518, -86.433965, 31.002979], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008605_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.434367, 30.934518], [-86.433965, 31.002694], [-86.50313, 31.002979], [-86.503483, 30.934802], [-86.434367, 30.934518]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [547433, 3422482, 554036, 3430038], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6603], "eo:cloud_cover": 35, "proj:transform": [1, 0, 547433, 0, -1, 3430038, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0068", "bbox": [-86.316114, 30.934419, -86.246339, 31.003078], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008606_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.246875, 30.934419], [-86.246339, 31.002692], [-86.315627, 31.003078], [-86.316114, 30.934803], [-86.246875, 30.934419]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [565333, 3422577, 571948, 3430144], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7567, 6615], "eo:cloud_cover": 22, "proj:transform": [1, 0, 565333, 0, -1, 3430144, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0069", "bbox": [-86.378574, 30.934452, -86.308884, 31.003042], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008606_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.309376, 30.934452], [-86.308884, 31.00269], [-86.378132, 31.003042], [-86.378574, 30.934803], [-86.309376, 30.934452]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [559366, 3422542, 565977, 3430105], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6611], "eo:cloud_cover": 97, "proj:transform": [1, 0, 559366, 0, -1, 3430105, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0070", "bbox": [-86.191196, 30.934361, -86.121261, 31.00314], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008607_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.121886, 30.934361], [-86.121261, 31.002686], [-86.19062, 31.00314], [-86.191196, 30.934813], [-86.121886, 30.934361]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [577267, 3422658, 583889, 3430231], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6622], "eo:cloud_cover": 32, "proj:transform": [1, 0, 577267, 0, -1, 3430231, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0071", "bbox": [-86.253655, 30.934391, -86.183805, 31.00311], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008607_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.184386, 30.934391], [-86.183805, 31.002691], [-86.253123, 31.00311], [-86.253655, 30.93481], [-86.184386, 30.934391]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [571300, 3422616, 577918, 3430186], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7570, 6618], "eo:cloud_cover": 68, "proj:transform": [1, 0, 571300, 0, -1, 3430186, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0072", "bbox": [-86.06628, 30.9343, -85.996173, 31.003207], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008608_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.996889, 30.9343], [-85.996173, 31.002686], [-86.065614, 31.003207], [-86.06628, 30.93482], [-85.996889, 30.9343]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [589201, 3422752, 595831, 3430332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6630], "eo:cloud_cover": 30, "proj:transform": [1, 0, 589201, 0, -1, 3430332, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0073", "bbox": [-86.128738, 30.934327, -86.058717, 31.003175], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008608_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.059387, 30.934327], [-86.058717, 31.002688], [-86.128117, 31.003175], [-86.128738, 30.934813], [-86.059387, 30.934327]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [583234, 3422703, 589860, 3430280], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6626], "eo:cloud_cover": 3, "proj:transform": [1, 0, 583234, 0, -1, 3430280, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0074", "bbox": [-85.816455, 30.934167, -85.746007, 31.00333], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008502_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.746902, 30.934167], [-85.746007, 31.002673], [-85.815609, 31.00333], [-85.816455, 30.934822], [-85.746902, 30.934167]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [613069, 3422979, 619715, 3430573], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7594, 6646], "eo:cloud_cover": 52, "proj:transform": [1, 0, 613069, 0, -1, 3430573, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0075", "bbox": [-86.003823, 30.934269, -85.933631, 31.003236], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008501_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.934391, 30.934269], [-85.933631, 31.002681], [-86.003112, 31.003236], [-86.003823, 30.934823], [-85.934391, 30.934269]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [595168, 3422804, 601802, 3430387], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7583, 6634], "eo:cloud_cover": 8, "proj:transform": [1, 0, 595168, 0, -1, 3430387, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0076", "bbox": [-85.941366, 30.934235, -85.871089, 31.00327], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008501_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.871894, 30.934235], [-85.871089, 31.002681], [-85.940611, 31.00327], [-85.941366, 30.934823], [-85.871894, 30.934235]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [601135, 3422859, 607773, 3430446], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7587, 6638], "eo:cloud_cover": 59, "proj:transform": [1, 0, 601135, 0, -1, 3430446, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0077", "bbox": [-86.753316, 30.934648, -86.684141, 31.002851], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008603_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.684365, 30.934648], [-86.684141, 31.0027], [-86.753141, 31.002851], [-86.753316, 30.934798], [-86.684365, 30.934648]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [523566, 3422402, 530153, 3429944], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7542, 6587], "eo:cloud_cover": 64, "proj:transform": [1, 0, 523566, 0, -1, 3429944, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0078", "bbox": [-86.690855, 30.934615, -86.621594, 31.002887], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008603_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.621863, 30.934615], [-86.621594, 31.002704], [-86.690635, 31.002887], [-86.690855, 30.934799], [-86.621863, 30.934615]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [529533, 3422417, 536124, 3429963], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7546, 6591], "eo:cloud_cover": 21, "proj:transform": [1, 0, 529533, 0, -1, 3429963, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0079", "bbox": [-85.629082, 30.934072, -85.558378, 31.003423], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008504_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.559409, 30.934072], [-85.558378, 31.002663], [-85.628101, 31.003423], [-85.629082, 30.934829], [-85.559409, 30.934072]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [630971, 3423185, 637629, 3430789], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7604, 6658], "eo:cloud_cover": 65, "proj:transform": [1, 0, 630971, 0, -1, 3430789, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0080", "bbox": [-85.566619, 30.934046, -85.49583, 31.003456], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008504_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.496906, 30.934046], [-85.49583, 31.002662], [-85.565593, 31.003456], [-85.566619, 30.934838], [-85.496906, 30.934046]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [636939, 3423261, 643601, 3430868], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7607, 6662], "eo:cloud_cover": 95, "proj:transform": [1, 0, 636939, 0, -1, 3430868, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0081", "bbox": [-85.753989, 30.934141, -85.683456, 31.003364], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008503_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.684396, 30.934141], [-85.683456, 31.002673], [-85.753099, 31.003364], [-85.753989, 30.934831], [-85.684396, 30.934141]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [619037, 3423045, 625687, 3430642], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7597, 6650], "eo:cloud_cover": 39, "proj:transform": [1, 0, 619037, 0, -1, 3430642, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0082", "bbox": [-85.691535, 30.934103, -85.620917, 31.003395], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008503_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.621902, 30.934103], [-85.620917, 31.002669], [-85.6906, 31.003395], [-85.691535, 30.934827], [-85.621902, 30.934103]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [625004, 3423113, 631658, 3430714], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7601, 6654], "eo:cloud_cover": 26, "proj:transform": [1, 0, 625004, 0, -1, 3430714, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0001", "bbox": [-85.441706, 30.933975, -85.370747, 31.003522], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_ne_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008505_ne_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_ne_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.371913, 30.933975], [-85.370747, 31.00266], [-85.440589, 31.003522], [-85.441706, 30.934836], [-85.371913, 30.933975]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [648874, 3423421, 655544, 3431036], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7615, 6670], "eo:cloud_cover": 89, "proj:transform": [1, 0, 648874, 0, -1, 3431036, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0002", "bbox": [-85.504167, 30.934008, -85.433293, 31.003486], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008505_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.434414, 30.934008], [-85.433293, 31.002658], [-85.503096, 31.003486], [-85.504167, 30.934834], [-85.434414, 30.934008]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [642906, 3423339, 649572, 3430950], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7611, 6666], "eo:cloud_cover": 33, "proj:transform": [1, 0, 642906, 0, -1, 3430950, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0083", "bbox": [-85.87891, 30.934198, -85.808547, 31.003302], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008502_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.809397, 30.934198], [-85.808547, 31.002679], [-85.87811, 31.003302], [-85.87891, 30.934819], [-85.809397, 30.934198]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [607102, 3422917, 613744, 3430508], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7591, 6642], "eo:cloud_cover": 26, "proj:transform": [1, 0, 607102, 0, -1, 3430508, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0086", "bbox": [-87.816197, 30.87189, -87.746352, 30.940609], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_se_16_1_20110801.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_se_16_1_20110801.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_se_16_1_20110801.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746352, 30.872308], [-87.746882, 30.940609], [-87.816197, 30.94019], [-87.815618, 30.87189], [-87.746352, 30.872308]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2011", "proj:bbox": [422031, 3415689, 428653, 3423259], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7570, 6622], "eo:cloud_cover": 40, "proj:transform": [1, 0, 422031, 0, -1, 3423259, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0087", "bbox": [-87.753661, 30.934419, -87.683886, 31.003078], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_nw_16_1_20110801.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_nw_16_1_20110801.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_nw_16_1_20110801.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683886, 30.934803], [-87.684373, 31.003078], [-87.753661, 31.002692], [-87.753125, 30.934419], [-87.683886, 30.934803]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2011", "proj:bbox": [428052, 3422577, 434667, 3430144], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7567, 6615], "eo:cloud_cover": 36, "proj:transform": [1, 0, 428052, 0, -1, 3430144, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0088", "bbox": [-87.753652, 30.871926, -87.683891, 30.940576], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_sw_16_1_20110801.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_sw_16_1_20110801.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_sw_16_1_20110801.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683891, 30.87231], [-87.684377, 30.940576], [-87.753652, 30.94019], [-87.753117, 30.871926], [-87.683891, 30.87231]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2011", "proj:bbox": [428006, 3415651, 434624, 3423217], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6618], "eo:cloud_cover": 23, "proj:transform": [1, 0, 428006, 0, -1, 3423217, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0089", "bbox": [-87.816182, 30.55939, -87.746368, 30.6281], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_ne_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_ne_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_ne_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746368, 30.559805], [-87.746892, 30.6281], [-87.816182, 30.627684], [-87.815609, 30.55939], [-87.746368, 30.559805]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421780, 3381056, 428421, 3388625], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6641], "eo:cloud_cover": 36, "proj:transform": [1, 0, 421780, 0, -1, 3388625, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0090", "bbox": [-87.81619, 30.809396, -87.746349, 30.878106], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_ne_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_ne_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_ne_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746349, 30.809814], [-87.746878, 30.878106], [-87.81619, 30.877688], [-87.815612, 30.809396], [-87.746349, 30.809814]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421981, 3408763, 428607, 3416332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6626], "eo:cloud_cover": 84, "proj:transform": [1, 0, 421981, 0, -1, 3416332, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0091", "bbox": [-87.816185, 30.621895, -87.746357, 30.690605], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_se_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_se_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_se_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746357, 30.62231], [-87.746882, 30.690605], [-87.816185, 30.690188], [-87.815611, 30.621895], [-87.746357, 30.62231]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421830, 3387983, 428468, 3395552], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6638], "eo:cloud_cover": 31, "proj:transform": [1, 0, 421830, 0, -1, 3395552, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0092", "bbox": [-87.816194, 30.746893, -87.746358, 30.815603], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_se_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_se_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_se_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746358, 30.74731], [-87.746885, 30.815603], [-87.816194, 30.815185], [-87.815618, 30.746893], [-87.746358, 30.74731]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421930, 3401836, 428560, 3409405], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6630], "eo:cloud_cover": 24, "proj:transform": [1, 0, 421930, 0, -1, 3409405, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0093", "bbox": [-87.753639, 30.559423, -87.683911, 30.628074], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_nw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_nw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_nw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683911, 30.559805], [-87.68439, 30.628074], [-87.753639, 30.627692], [-87.753111, 30.559423], [-87.683911, 30.559805]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427774, 3381018, 434411, 3388584], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6637], "eo:cloud_cover": 61, "proj:transform": [1, 0, 427774, 0, -1, 3388584, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0094", "bbox": [-87.753654, 30.809423, -87.683898, 30.878073], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_nw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_nw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_nw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683898, 30.809806], [-87.684382, 30.878073], [-87.753654, 30.877688], [-87.75312, 30.809423], [-87.683898, 30.809806]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427959, 3408724, 434581, 3416290], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6622], "eo:cloud_cover": 29, "proj:transform": [1, 0, 427959, 0, -1, 3416290, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0095", "bbox": [-87.753646, 30.746928, -87.683895, 30.815579], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_sw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_sw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_sw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683895, 30.747312], [-87.684378, 30.815579], [-87.753646, 30.815194], [-87.753114, 30.746928], [-87.683895, 30.747312]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427913, 3401798, 434539, 3409364], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6626], "eo:cloud_cover": 63, "proj:transform": [1, 0, 427913, 0, -1, 3409364, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0096", "bbox": [-87.753639, 30.684424, -87.683903, 30.753075], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_nw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_nw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_nw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683903, 30.684807], [-87.684385, 30.753075], [-87.753639, 30.752691], [-87.753109, 30.684424], [-87.683903, 30.684807]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427867, 3394871, 434496, 3402437], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6629], "eo:cloud_cover": 26, "proj:transform": [1, 0, 427867, 0, -1, 3402437, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0097", "bbox": [-87.816189, 30.68439, -87.746357, 30.753109], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_ne_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_ne_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_ne_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746357, 30.684806], [-87.746883, 30.753109], [-87.816189, 30.752692], [-87.815614, 30.68439], [-87.746357, 30.684806]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421880, 3394909, 428514, 3402479], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7570, 6634], "eo:cloud_cover": 1, "proj:transform": [1, 0, 421880, 0, -1, 3402479, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0098", "bbox": [-87.753644, 30.621929, -87.683901, 30.69057], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_sw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_sw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_sw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683901, 30.622311], [-87.684382, 30.69057], [-87.753644, 30.690187], [-87.753115, 30.621929], [-87.683901, 30.622311]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427820, 3387945, 434454, 3395510], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7565, 6634], "eo:cloud_cover": 73, "proj:transform": [1, 0, 427820, 0, -1, 3395510, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0099", "bbox": [-87.753635, 30.496927, -87.683911, 30.565569], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_sw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_sw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_sw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683911, 30.497308], [-87.684389, 30.565569], [-87.753635, 30.565186], [-87.753109, 30.496927], [-87.683911, 30.497308]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427728, 3374092, 434369, 3381657], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7565, 6641], "eo:cloud_cover": 90, "proj:transform": [1, 0, 427728, 0, -1, 3381657, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0084", "bbox": [-85.316796, 30.93392, -85.245667, 31.003585], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_ne_16_1_20110802.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_ne_16_1_20110802.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_ne_16_1_20110802.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.246924, 30.93392], [-85.245667, 31.002654], [-85.315589, 31.003585], [-85.316796, 30.934848], [-85.246924, 30.93392]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-02T00:00:00Z", "naip:year": "2011", "proj:bbox": [660809, 3423596, 667487, 3431217], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7621, 6678], "eo:cloud_cover": 52, "proj:transform": [1, 0, 660809, 0, -1, 3431217, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0085", "bbox": [-87.816195, 30.934391, -87.746345, 31.00311], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_ne_16_1_20110801.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_ne_16_1_20110801.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_ne_16_1_20110801.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746345, 30.93481], [-87.746877, 31.00311], [-87.816195, 31.002691], [-87.815614, 30.934391], [-87.746345, 30.93481]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2012", "proj:bbox": [422082, 3422616, 428700, 3430186], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "xx", "proj:shape": [7570, 6618], "eo:cloud_cover": 3, "proj:transform": [1, 0, 422082, 0, -1, 3430186, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0100", "bbox": [-87.816179, 30.496894, -87.74637, 30.565604], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_se_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_se_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_se_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.74637, 30.497308], [-87.746892, 30.565604], [-87.816179, 30.565188], [-87.815608, 30.496894], [-87.74637, 30.497308]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": 2013, "proj:bbox": [421730, 3374130, 428375, 3381699], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "zz", "proj:shape": [7569, 6645], "eo:cloud_cover": 50, "proj:transform": [1, 0, 421730, 0, -1, 3381699, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} ================================================ FILE: src/pgstac/tests/testdata/items.pgcopy ================================================ pgstac-test-item-0003 0103000020E610000001000000050000005B3FFD67CD5355C0C4211B4817EF3E400CE6AF90B95355C0A112D731AE003F4004C93B87325855C0BEBC00FBE8003F40FA0AD28C455855C000E5EFDE51EF3E405B3FFD67CD5355C0C4211B4817EF3E40 pgstac-test-collection 2011-08-25 00:00:00+00 2011-08-25 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0004 0103000020E610000001000000050000006364C91CCBC755C0DE76A1B94EEF3E4062307F85CCC755C002A08A1BB7003F403E7958A835CC55C0E75608ABB1003F40E695EB6D33CC55C0C32D1F4949EF3E406364C91CCBC755C0DE76A1B94EEF3E40 pgstac-test-collection 2011-08-24 00:00:00+00 2011-08-24 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_ne_16_1_20110824.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008707_ne_16_1_20110824.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_ne_16_1_20110824.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [481788, 3422382, 488367, 3429918], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7536, 6579], "eo:cloud_cover": 23, "proj:transform": [1, 0, 481788, 0, -1, 3429918, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0005 0103000020E61000000100000005000000C1525DC0CBC355C03D7FDAA84EEF3E40D97A8670CCC355C0C7D5C8AEB4003F40AF06280D35C855C06478EC67B1003F402785798F33C855C0D921FE614BEF3E40C1525DC0CBC355C03D7FDAA84EEF3E40 pgstac-test-collection 2011-08-24 00:00:00+00 2011-08-24 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_nw_16_1_20110824.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008708_nw_16_1_20110824.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_nw_16_1_20110824.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [487758, 3422377, 494334, 3429909], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7532, 6576], "eo:cloud_cover": 3, "proj:transform": [1, 0, 487758, 0, -1, 3429909, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0006 0103000020E61000000100000005000000F20A444FCACB55C03D7FDAA84EEF3E40C138B874CCCB55C054C6BFCFB8003F40DE567A6D36D055C0E199D024B1003F409ECF807A33D055C0CA52EBFD46EF3E40F20A444FCACB55C03D7FDAA84EEF3E40 pgstac-test-collection 2011-08-24 00:00:00+00 2011-08-24 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_nw_16_1_20110824.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008707_nw_16_1_20110824.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_nw_16_1_20110824.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [475817, 3422390, 482401, 3429929], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7539, 6584], "eo:cloud_cover": 100, "proj:transform": [1, 0, 475817, 0, -1, 3429929, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0007 0103000020E61000000100000005000000CF68AB92C8D755C01F662FDB4E9F3E40850662D9CCD755C0F8AA9509BFB03E405A2A6F4738DC55C05EBBB4E1B0B03E401BF1643733DC55C086764EB3409F3E40CF68AB92C8D755C01F662FDB4E9F3E40 pgstac-test-collection 2011-08-17 00:00:00+00 2011-08-17 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_se_16_1_20110817.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_se_16_1_20110817.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_se_16_1_20110817.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [457770, 3387803, 464384, 3395352], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7549, 6614], "eo:cloud_cover": 59, "proj:transform": [1, 0, 457770, 0, -1, 3395352, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0008 0103000020E61000000100000005000000CF8250DEC7DB55C0433C122F4F9F3E406EC493DDCCDB55C00E9F7422C1B03E405A10CAFB38E055C0BDC3EDD0B0B03E404B75012F33E055C0F2608BDD3E9F3E40CF8250DEC7DB55C0433C122F4F9F3E40 pgstac-test-collection 2011-08-17 00:00:00+00 2011-08-17 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_sw_16_1_20110817.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_sw_16_1_20110817.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_sw_16_1_20110817.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [451780, 3387825, 458398, 3395377], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7552, 6618], "eo:cloud_cover": 64, "proj:transform": [1, 0, 451780, 0, -1, 3395377, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0009 0103000020E61000000100000005000000FF06EDD5C7DB55C06155BDFC4EAF3E409D4830D5CCDB55C02CB81FF0C0C03E405A10CAFB38E055C03BE5D18DB0C03E403333333333E055C0107A36AB3EAF3E40FF06EDD5C7DB55C06155BDFC4EAF3E40 pgstac-test-collection 2011-08-17 00:00:00+00 2011-08-17 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_nw_16_1_20110817.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_nw_16_1_20110817.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_nw_16_1_20110817.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [451811, 3394751, 458425, 3402303], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7552, 6614], "eo:cloud_cover": 61, "proj:transform": [1, 0, 451811, 0, -1, 3402303, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0010 0103000020E61000000100000005000000D53F8864C8D755C03D7FDAA84EAF3E408BDD3EABCCD755C015C440D7BEC03E402BA6D24F38DC55C07CD45FAFB0C03E40EC6CC83F33DC55C04487C09140AF3E40D53F8864C8D755C03D7FDAA84EAF3E40 pgstac-test-collection 2011-08-17 00:00:00+00 2011-08-17 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_ne_16_1_20110817.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_ne_16_1_20110817.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_ne_16_1_20110817.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [457797, 3394729, 464408, 3402278], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7549, 6611], "eo:cloud_cover": 31, "proj:transform": [1, 0, 457797, 0, -1, 3402278, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0011 0103000020E61000000100000005000000931CB0ABC9CF55C0C05DF6EB4EEF3E404AEEB089CCCF55C0CAC2D7D7BA003F406DC9AA0837D455C0FFB27BF2B0003F40459E245D33D455C09545611745EF3E40931CB0ABC9CF55C0C05DF6EB4EEF3E40 pgstac-test-collection 2011-08-17 00:00:00+00 2011-08-17 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_ne_16_1_20110817.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008706_ne_16_1_20110817.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_ne_16_1_20110817.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [469847, 3422402, 476434, 3429944], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7542, 6587], "eo:cloud_cover": 41, "proj:transform": [1, 0, 469847, 0, -1, 3429944, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0012 0103000020E61000000100000005000000C3D4963AC8D755C01F662FDB4EEF3E4032ACE28DCCD755C0BC783F6EBF003F40B45BCB6438DC55C082919735B1003F40D42AFA4333DC55C0E57E87A240EF3E40C3D4963AC8D755C01F662FDB4EEF3E40 pgstac-test-collection 2011-08-17 00:00:00+00 2011-08-17 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_ne_16_1_20110817.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008705_ne_16_1_20110817.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_ne_16_1_20110817.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [457906, 3422435, 464501, 3429985], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7550, 6595], "eo:cloud_cover": 4, "proj:transform": [1, 0, 457906, 0, -1, 3429985, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0013 0103000020E61000000100000005000000342E1C08C9D355C06155BDFC4EEF3E40BB61DBA2CCD355C06495D233BD003F400DA7CCCD37D855C082919735B1003F40151A886533D855C0DE59BBED42EF3E40342E1C08C9D355C06155BDFC4EEF3E40 pgstac-test-collection 2011-08-17 00:00:00+00 2011-08-17 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_nw_16_1_20110817.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008706_nw_16_1_20110817.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_nw_16_1_20110817.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [463876, 3422417, 470467, 3429963], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7546, 6591], "eo:cloud_cover": 2, "proj:transform": [1, 0, 463876, 0, -1, 3429963, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0014 0103000020E6100000010000000500000089F02F82C6E355C06155BDFC4E7F3E4026FE28EACCE355C0B9A81611C5903E40EE3F321D3AE855C0F9F5436CB0903E40C896E5EB32E855C0A1A2EA573A7F3E4089F02F82C6E355C06155BDFC4E7F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439724, 3374025, 446357, 3381584], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6633], "eo:cloud_cover": 17, "proj:transform": [1, 0, 439724, 0, -1, 3381584, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0015 0103000020E61000000100000005000000D97A8670CCB755C0D921FE614BEF3E4051F9D7F2CAB755C06478EC67B1003F402785798F33BC55C0C7D5C8AEB4003F403FADA23F34BC55C03D7FDAA84EEF3E40D97A8670CCB755C0D921FE614BEF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008601_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [505666, 3422377, 512242, 3429909], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7532, 6576], "eo:cloud_cover": 54, "proj:transform": [1, 0, 505666, 0, -1, 3429909, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0016 0103000020E6100000010000000500000091B41B7DCCBB55C00E2F88484DEF3E40D9942BBCCBBB55C0A5677A89B1003F40868DB27E33C055C051D9B0A6B2003F40CE531D7233C055C019A9F7544EEF3E4091B41B7DCCBB55C00E2F88484DEF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008601_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [499700, 3422375, 506271, 3429904], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7529, 6571], "eo:cloud_cover": 13, "proj:transform": [1, 0, 499700, 0, -1, 3429904, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0017 0103000020E6100000010000000500000062307F85CCAF55C0CA52EBFD46EF3E4022A98592C9AF55C0E199D024B1003F403FC7478B33B455C054C6BFCFB8003F400EF5BBB035B455C03D7FDAA84EEF3E4062307F85CCAF55C0CA52EBFD46EF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008602_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [517599, 3422390, 524183, 3429929], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7539, 6584], "eo:cloud_cover": 59, "proj:transform": [1, 0, 517599, 0, -1, 3429929, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0018 0103000020E610000001000000050000001A6A1492CCB355C0C32D1F4949EF3E40C286A757CAB355C0E75608ABB1003F409ECF807A33B855C002A08A1BB7003F409D9B36E334B855C0DE76A1B94EEF3E401A6A1492CCB355C0C32D1F4949EF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008602_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [511633, 3422382, 518212, 3429918], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7536, 6579], "eo:cloud_cover": 29, "proj:transform": [1, 0, 511633, 0, -1, 3429918, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0019 0103000020E610000001000000050000001D3A3DEFC6DF55C0433C122F4FEF3E404417D4B7CCDF55C02593533BC3003F400C59DDEA39E455C03BE5D18DB0003F407522C15433E455C0F98557923CEF3E401D3A3DEFC6DF55C0433C122F4FEF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008704_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445964, 3422482, 452567, 3430038], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6603], "eo:cloud_cover": 52, "proj:transform": [1, 0, 445964, 0, -1, 3430038, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0020 0103000020E61000000100000005000000D68D7747C6E355C0070ABC934FEF3E40E50E9BC8CCE355C0DC7EF964C5003F40CA4FAA7D3AE855C03BE5D18DB0003F4063B7CF2A33E855C006685BCD3AEF3E40D68D7747C6E355C0070ABC934FEF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008704_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439994, 3422511, 446600, 3430070], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6606], "eo:cloud_cover": 39, "proj:transform": [1, 0, 439994, 0, -1, 3430070, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0021 0103000020E610000001000000050000008FC70C54C6E355C0252367614FDF3E409D4830D5CCE355C0FA97A432C5F03E40E29178793AE855C058FE7C5BB0F03E4063B7CF2A33E855C02481069B3ADF3E408FC70C54C6E355C0252367614FDF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008704_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439955, 3415584, 446565, 3423143], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6610], "eo:cloud_cover": 29, "proj:transform": [1, 0, 439955, 0, -1, 3423143, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0022 0103000020E610000001000000050000007C28D192C7DB55C0A2444B1E4FEF3E40BB61DBA2CCDB55C032755776C1003F406C7BBB2539E055C09FAA4203B1003F40A4A65D4C33E055C0107A36AB3EEF3E407C28D192C7DB55C0A2444B1E4FEF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008705_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [451935, 3422457, 458534, 3430010], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7553, 6599], "eo:cloud_cover": 84, "proj:transform": [1, 0, 451935, 0, -1, 3430010, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0023 0103000020E6100000010000000500000032ACE28DCCBF55C019A9F7544EEF3E407A724D81CCBF55C051D9B0A6B2003F40276BD44334C455C0A5677A89B1003F406F4BE48233C455C00E2F88484DEF3E4032ACE28DCCBF55C019A9F7544EEF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008708_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [493729, 3422375, 500300, 3429904], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7529, 6571], "eo:cloud_cover": 56, "proj:transform": [1, 0, 493729, 0, -1, 3429904, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0024 0103000020E6100000010000000500000001BD70E7C2F755C048F949B54FCF3E407F2F8507CDF755C0EF3A1BF2CFE03E40E6E61BD13DFC55C0D61F6118B0E03E40105D50DF32FC55C0D0D556EC2FCF3E4001BD70E7C2F755C048F949B54FCF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [410024, 3408849, 416657, 3416426], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6633], "eo:cloud_cover": 58, "proj:transform": [1, 0, 410024, 0, -1, 3416426, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0025 0103000020E61000000100000005000000A9A5B915C2FB55C0ADBEBA2A50CF3E40F6798CF2CCFB55C0A626C11BD2E03E40B648DA8D3E0056C035289A07B0E03E40F81A82E3320056C07DAF21382ECF3E40A9A5B915C2FB55C0ADBEBA2A50CF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [404045, 3408898, 410683, 3416478], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6638], "eo:cloud_cover": 65, "proj:transform": [1, 0, 404045, 0, -1, 3416478, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0026 0103000020E61000000100000005000000EA7AA2EBC2F755C0ADBEBA2A50BF3E407F2F8507CDF755C0AD4B8DD0CFD03E4046EF54C03DFC55C09430D3F6AFD03E403FE1ECD632FC55C0359BC76130BF3E40EA7AA2EBC2F755C0ADBEBA2A50BF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409966, 3401923, 416603, 3409499], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6637], "eo:cloud_cover": 52, "proj:transform": [1, 0, 409966, 0, -1, 3409499, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0027 0103000020E6100000010000000500000061DF4E22C2FB55C06CCF2C0950BF3E40DF37BEF6CCFB55C0653733FAD1D03E40E6CC76853E0056C09430D3F6AFD03E40F81A82E3320056C03CC093162EBF3E4061DF4E22C2FB55C06CCF2C0950BF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [403983, 3401971, 410625, 3409551], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6642], "eo:cloud_cover": 43, "proj:transform": [1, 0, 403983, 0, -1, 3409551, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0028 0103000020E6100000010000000500000060AB048BC3F355C0A80183A44FCF3E40F6798CF2CCF355C055682096CDE03E4040321D3A3DF855C0F3380CE6AFE03E40390A100533F855C087C1FC1532CF3E4060AB048BC3F355C0A80183A44FCF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [416002, 3408804, 422632, 3416377], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6630], "eo:cloud_cover": 46, "proj:transform": [1, 0, 416002, 0, -1, 3416377, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0029 0103000020E6100000010000000500000060AB048BC3F355C06CCF2C0950BF3E4026FE28EACCF355C07381CB63CDD03E40B77C24253DF855C0B2497EC4AFD03E4081D07AF832F855C04B8FA67A32BF3E4060AB048BC3F355C06CCF2C0950BF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415948, 3401878, 422582, 3409450], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7572, 6634], "eo:cloud_cover": 42, "proj:transform": [1, 0, 415948, 0, -1, 3409450, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0030 0103000020E61000000100000005000000A723809BC5E755C0252367614FCF3E40B58AFED0CCE755C0946A9F8EC7E03E407104A9143BEC55C03BE5D18DB0E03E40F243A51133EC55C06C95607138CF3E40A723809BC5E755C0252367614FCF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433938, 3408689, 440556, 3416252], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6618], "eo:cloud_cover": 16, "proj:transform": [1, 0, 433938, 0, -1, 3416252, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0031 0103000020E610000001000000050000001897AAB4C5E755C048F949B54FBF3E405682C5E1CCE755C0B2834A5CC7D03E4041800C1D3BEC55C058FE7C5BB0D03E40AA7D3A1E33EC55C0906B43C538BF3E401897AAB4C5E755C048F949B54FBF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433895, 3401763, 440517, 3409325], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6622], "eo:cloud_cover": 16, "proj:transform": [1, 0, 433895, 0, -1, 3409325, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0032 0103000020E610000001000000050000003596B036C6E355C0A2444B1E4FCF3E405C59A2B3CCE355C0BF654E97C5E03E40E29178793AE855C01DCC26C0B0E03E404B75012F33E855C0A1A2EA573ACF3E403596B036C6E355C0A2444B1E4FCF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008712_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439916, 3408657, 446531, 3416217], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7560, 6615], "eo:cloud_cover": 7, "proj:transform": [1, 0, 439916, 0, -1, 3416217, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0033 0103000020E61000000100000005000000766B990CC7DF55C0C05DF6EB4EBF3E40E50E9BC8CCDF55C0E4A3C519C3D03E40897AC1A739E455C09AED0A7DB0D03E40AA7D3A1E33E455C0179F02603CBF3E40766B990CC7DF55C0C05DF6EB4EBF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008712_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445860, 3401702, 452474, 3409258], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6614], "eo:cloud_cover": 49, "proj:transform": [1, 0, 445860, 0, -1, 3409258, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0034 0103000020E61000000100000005000000BE4BA94BC6E355C0252367614FBF3E40149337C0CCE355C03C873254C5D03E40D026874F3AE855C03BE5D18DB0D03E4021C8410933E855C0C478CDAB3ABF3E40BE4BA94BC6E355C0252367614FBF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008712_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439878, 3401731, 446496, 3409290], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6618], "eo:cloud_cover": 3, "proj:transform": [1, 0, 439878, 0, -1, 3409290, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0035 0103000020E61000000100000005000000BAF605F4C2F755C02AE09EE74FAF3E407F2F8507CDF755C0CB64389ECFC03E408DB5BFB33DFC55C0F3380CE6AFC03E405723BBD232FC55C0F3AB394030AF3E40BAF605F4C2F755C02AE09EE74FAF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409908, 3394996, 416549, 3402572], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6641], "eo:cloud_cover": 33, "proj:transform": [1, 0, 409908, 0, -1, 3402572, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0036 0103000020E610000001000000050000001A19E42EC2FB55C02AE09EE74FAF3E40C7F5EFFACCFB55C02448A5D8D1C03E401651137D3E0056C0F3380CE6AFC03E40F81A82E3320056C0FAD005F52DAF3E401A19E42EC2FB55C02AE09EE74FAF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [403921, 3395044, 410567, 3402624], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6646], "eo:cloud_cover": 12, "proj:transform": [1, 0, 403921, 0, -1, 3402624, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0037 0103000020E6100000010000000500000073309B00C3F755C048F949B54F9F3E4050ABE80FCDF755C0E97DE36BCFB03E40A5F78DAF3DFC55C01152B7B3AFB03E403FE1ECD632FC55C011C5E40D309F3E4073309B00C3F755C048F949B54F9F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409850, 3388069, 416495, 3395645], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6645], "eo:cloud_cover": 54, "proj:transform": [1, 0, 409850, 0, -1, 3395645, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0038 0103000020E61000000100000005000000A3CEDC43C2FB55C04EB6813B509F3E407F2F8507CDFB55C0E25817B7D1B03E4004E621533E0056C0B2497EC4AFB03E40B62BF4C1320056C05F96766A2E9F3E40A3CEDC43C2FB55C04EB6813B509F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [403860, 3388118, 410509, 3395697], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7579, 6649], "eo:cloud_cover": 77, "proj:transform": [1, 0, 403860, 0, -1, 3395697, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0039 0103000020E610000001000000050000005AD427B9C3F355C0E9F010C64FAF3E4038691A14CDF355C0374F75C8CDC03E4016855D143DF855C0170FEF39B0C03E40B05417F032F855C069A8514832AF3E405AD427B9C3F355C0E9F010C64FAF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415894, 3394951, 422531, 3402524], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6637], "eo:cloud_cover": 12, "proj:transform": [1, 0, 415894, 0, -1, 3402524, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0040 0103000020E610000001000000050000002B508BC1C3F355C06612F5824F9F3E4038691A14CDF355C055682096CDB03E405E4BC8073DF855C035289A07B0B03E40E0D8B3E732F855C087C1FC15329F3E402B508BC1C3F355C06612F5824F9F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415840, 3388024, 422481, 3395597], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6641], "eo:cloud_cover": 10, "proj:transform": [1, 0, 415840, 0, -1, 3395597, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0041 0103000020E61000000100000005000000779FE3A3C5E755C0252367614FAF3E40CDCCCCCCCCE755C0D5592DB0C7C03E4000917EFB3AEC55C07CD45FAFB0C03E40514CDE0033EC55C00D8D278238AF3E40779FE3A3C5E755C0252367614FAF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433853, 3394836, 440479, 3402399], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6626], "eo:cloud_cover": 3, "proj:transform": [1, 0, 433853, 0, -1, 3402399, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0042 0103000020E61000000100000005000000D0D03FC1C5E755C0A80183A44F9F3E4026FE28EACCE755C0B2834A5CC7B03E40B8CA13083BEC55C0F9F5436CB0B03E40DA01D71533EC55C0906B43C5389F3E40D0D03FC1C5E755C0A80183A44F9F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433810, 3387910, 440440, 3395472], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6630], "eo:cloud_cover": 99, "proj:transform": [1, 0, 433810, 0, -1, 3395472, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0043 0103000020E610000001000000050000002FA52E19C7DF55C0A2444B1E4FAF3E409D4830D5CCDF55C06682E15CC3C03E40897AC1A739E455C0BDC3EDD0B0C03E40923B6C2233E455C09A7D1EA33CAF3E402FA52E19C7DF55C0A2444B1E4FAF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445825, 3394776, 452443, 3402332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6618], "eo:cloud_cover": 80, "proj:transform": [1, 0, 445825, 0, -1, 3402332, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0044 0103000020E610000001000000050000004701A260C6E355C0014D840D4FAF3E40B58AFED0CCE355C0BF654E97C5C03E40B8E4B8533AE855C05EBBB4E1B0C03E40F243A51133E855C0A1A2EA573AAF3E404701A260C6E355C0014D840D4FAF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439839, 3394804, 446461, 3402364], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7560, 6622], "eo:cloud_cover": 30, "proj:transform": [1, 0, 439839, 0, -1, 3402364, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0045 0103000020E61000000100000005000000B85A272EC7DF55C0842BA0504F9F3E405682C5E1CCDF55C043ACFE08C3B03E407138F3AB39E455C09AED0A7DB0B03E4063B7CF2A33E455C07C6473D53C9F3E40B85A272EC7DF55C0842BA0504F9F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445790, 3387850, 452412, 3395405], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7555, 6622], "eo:cloud_cover": 82, "proj:transform": [1, 0, 445790, 0, -1, 3395405, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0046 0103000020E61000000100000005000000D0B69A75C6E355C0E333D93F4F9F3E403E40F7E5CCE355C09B8F6B43C5B03E4089601C5C3AE855C03BE5D18DB0B03E40AA7D3A1E33E855C02481069B3A9F3E40D0B69A75C6E355C0E333D93F4F9F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439800, 3387878, 446426, 3395437], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6626], "eo:cloud_cover": 16, "proj:transform": [1, 0, 439800, 0, -1, 3395437, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0047 0103000020E61000000100000005000000FBE59315C3F755C0CBD765F84F8F3E4020274C18CDF755C0AD4B8DD0CFA03E40A5F78DAF3DFC55C0D61F6118B0A03E40279F1EDB32FC55C0359BC761308F3E40FBE59315C3F755C0CBD765F84F8F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008725_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409792, 3381143, 416441, 3388719], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6649], "eo:cloud_cover": 43, "proj:transform": [1, 0, 409792, 0, -1, 3388719, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0048 0103000020E61000000100000005000000849B8C2AC3F755C0A80183A44F7F3E40C11E1329CDF755C08A75AA7CCF903E40938C9C853DFC55C0534145D5AF903E40E6AF90B932FC55C0B2BCAB1E307F3E40849B8C2AC3F755C0A80183A44F7F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008725_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409735, 3374216, 416387, 3381792], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6652], "eo:cloud_cover": 4, "proj:transform": [1, 0, 409735, 0, -1, 3381792, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0049 0103000020E61000000100000005000000FBCBEEC9C3F355C0E9F010C64F8F3E4020274C18CDF355C032923D42CDA03E408DCF64FF3CF855C0B2497EC4AFA03E40E0D8B3E732F855C00AA01859328F3E40FBCBEEC9C3F355C0E9F010C64F8F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415786, 3381098, 422431, 3388670], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7572, 6645], "eo:cloud_cover": 9, "proj:transform": [1, 0, 415786, 0, -1, 3388670, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0050 0103000020E61000000100000005000000B40584D6C3F355C06CCF2C09507F3E40F0A2AF20CDF355C055682096CD903E408DCF64FF3CF855C0D61F6118B0903E40C896E5EB32F855C02E76FBAC327F3E40B40584D6C3F355C06CCF2C09507F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415732, 3374172, 422381, 3381744], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7572, 6649], "eo:cloud_cover": 84, "proj:transform": [1, 0, 415732, 0, -1, 3381744, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0051 0103000020E6100000010000000500000012C0CDE2C5E755C0E333D93F4F8F3E407F2F8507CDE755C0946A9F8EC7A03E404757E9EE3AEC55C07CD45FAFB0A03E40514CDE0033EC55C0CB9D9960388F3E4012C0CDE2C5E755C0E333D93F4F8F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433768, 3380983, 440401, 3388546], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6633], "eo:cloud_cover": 22, "proj:transform": [1, 0, 433768, 0, -1, 3388546, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0052 0103000020E6100000010000000500000041446ADAC5E755C0252367614F7F3E40C7F5EFFACCE755C0D09CF529C7903E40D6E3BED53AEC55C058FE7C5BB0903E40C896E5EB32EC55C0AD84EE92387F3E4041446ADAC5E755C0252367614F7F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433726, 3374057, 440363, 3381619], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6637], "eo:cloud_cover": 12, "proj:transform": [1, 0, 433726, 0, -1, 3381619, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0053 0103000020E610000001000000050000005952EE3EC7DF55C01F662FDB4E8F3E40F6798CF2CCDF55C02593533BC3A03E4060CD018239E455C07CD45FAFB0A03E40390A100533E455C0B796C9703C8F3E405952EE3EC7DF55C01F662FDB4E8F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445756, 3380923, 452381, 3388479], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6625], "eo:cloud_cover": 24, "proj:transform": [1, 0, 445756, 0, -1, 3388479, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0054 0103000020E6100000010000000500000029E8F692C6E355C0C51A2E724F8F3E40C7F5EFFACCE355C01E6E8786C5A03E4047718E3A3AE855C0BDC3EDD0B0A03E40390A100533E855C006685BCD3A8F3E4029E8F692C6E355C0C51A2E724F8F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439762, 3380952, 446391, 3388511], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6629], "eo:cloud_cover": 5, "proj:transform": [1, 0, 439762, 0, -1, 3388511, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0055 0103000020E61000000100000005000000D09CF529C7DF55C06155BDFC4E7F3E40850662D9CCDF55C06682E15CC3903E403049658A39E455C05EBBB4E1B0903E40F243A51133E455C0F98557923C7F3E40D09CF529C7DF55C06155BDFC4E7F3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445721, 3373997, 452351, 3381553], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6630], "eo:cloud_cover": 79, "proj:transform": [1, 0, 445721, 0, -1, 3381553, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0056 0103000020E61000000100000005000000D80FB1C1C2F755C02AE09EE74FEF3E400EBC5AEECCF755C0302AA913D0003F40F8510DFB3DFC55C076172829B0003F40514CDE0033FC55C011C5E40D30EF3E40D80FB1C1C2F755C02AE09EE74FEF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [410140, 3422703, 416766, 3430280], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6626], "eo:cloud_cover": 62, "proj:transform": [1, 0, 410140, 0, -1, 3430280, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0057 0103000020E61000000100000005000000C0E78711C2FB55C08FA50F5D50EF3E40C7F5EFFACCFB55C0471E882CD2003F40E0F599B33E0056C035289A07B0003F4081D07AF8320056C01EA7E8482EEF3E40C0E78711C2FB55C08FA50F5D50EF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [404169, 3422752, 410799, 3430332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6630], "eo:cloud_cover": 7, "proj:transform": [1, 0, 404169, 0, -1, 3430332, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0058 0103000020E61000000100000005000000F0517FBDC2F755C08AE8D7D64FDF3E405682C5E1CCF755C09032E202D0F03E4087DEE2E13DFC55C0D61F6118B0F03E40B05417F032FC55C011C5E40D30DF3E40F0517FBDC2F755C08AE8D7D64FDF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [410082, 3415776, 416712, 3423353], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6630], "eo:cloud_cover": 53, "proj:transform": [1, 0, 410082, 0, -1, 3423353, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0059 0103000020E61000000100000005000000C0E78711C2FB55C0EFAD484C50DF3E40F6798CF2CCFB55C0471E882CD2F03E405740A19E3E0056C035289A07B0F03E40C896E5EB320056C01EA7E8482EDF3E40C0E78711C2FB55C0EFAD484C50DF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [404107, 3415825, 410741, 3423405], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6634], "eo:cloud_cover": 57, "proj:transform": [1, 0, 404107, 0, -1, 3423405, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0066 0103000020E610000001000000050000009D4830D5CC9755C006685BCD3AEF3E4036B05582C59755C03BE5D18DB0003F401BF16437339C55C0DC7EF964C5003F402A7288B8399C55C0070ABC934FEF3E409D4830D5CC9755C006685BCD3AEF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_ne_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008605_ne_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_ne_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [553400, 3422511, 560006, 3430070], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6606], "eo:cloud_cover": 86, "proj:transform": [1, 0, 553400, 0, -1, 3430070, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0060 0103000020E61000000100000005000000EF37DA71C3F355C02AE09EE74FEF3E405682C5E1CCF355C0374F75C8CD003F4010AE80423DF855C035289A07B0003F40390A100533F855C069A8514832EF3E40EF37DA71C3F355C02AE09EE74FEF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_nw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_nw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_nw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [416111, 3422658, 422733, 3430231], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6622], "eo:cloud_cover": 70, "proj:transform": [1, 0, 416111, 0, -1, 3430231, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0061 0103000020E6100000010000000500000037FE4465C3F355C0E9F010C64FDF3E40B58AFED0CCF355C0F65FE7A6CDF03E40B1A547533DF855C035289A07B0F03E40C2BF081A33F855C0C9B08A3732DF3E4037FE4465C3F355C0E9F010C64FDF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_sw_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_sw_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_sw_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [416056, 3415731, 422683, 3423304], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6627], "eo:cloud_cover": 94, "proj:transform": [1, 0, 416056, 0, -1, 3423304, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0062 0103000020E6100000010000000500000095B88E71C5E755C0E333D93F4FEF3E40749B70AFCCE755C0B2834A5CC7003F40826F9A3E3BEC55C0B806B64AB0003F404B75012F33EC55C08AAE0B3F38EF3E4095B88E71C5E755C0E333D93F4FEF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_ne_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_ne_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_ne_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [434023, 3422542, 440634, 3430105], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6611], "eo:cloud_cover": 85, "proj:transform": [1, 0, 434023, 0, -1, 3430105, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0063 0103000020E610000001000000050000001E6E8786C5E755C0A80183A44FDF3E40149337C0CCE755C0D09CF529C7F03E40826F9A3E3BEC55C076172829B0F03E401BF1643733EC55C04E7CB5A338DF3E401E6E8786C5E755C0A80183A44FDF3E40 pgstac-test-collection 2011-08-16 00:00:00+00 2011-08-16 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_se_16_1_20110816.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_se_16_1_20110816.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_se_16_1_20110816.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433980, 3415616, 440595, 3423178], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6615], "eo:cloud_cover": 2, "proj:transform": [1, 0, 433980, 0, -1, 3423178, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0064 0103000020E610000001000000050000005C59A2B3CC9F55C0107A36AB3EEF3E40948444DAC69F55C09FAA4203B1003F40459E245D33A455C032755776C1003F4084D72E6D38A455C0A2444B1E4FEF3E405C59A2B3CC9F55C0107A36AB3EEF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_ne_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008604_ne_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_ne_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [541466, 3422457, 548065, 3430010], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7553, 6599], "eo:cloud_cover": 40, "proj:transform": [1, 0, 541466, 0, -1, 3430010, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0065 0103000020E610000001000000050000002CD505BCCCA355C0E57E87A240EF3E404CA4349BC7A355C082919735B1003F40CE531D7233A855C0BC783F6EBF003F403D2B69C537A855C01F662FDB4EEF3E402CD505BCCCA355C0E57E87A240EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_nw_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008604_nw_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_nw_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [535499, 3422435, 542094, 3429985], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7550, 6595], "eo:cloud_cover": 27, "proj:transform": [1, 0, 535499, 0, -1, 3429985, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0067 0103000020E610000001000000050000008BDD3EABCC9B55C0F98557923CEF3E40F4A62215C69B55C03BE5D18DB0003F40BCE82B4833A055C02593533BC3003F40E3C5C21039A055C0433C122F4FEF3E408BDD3EABCC9B55C0F98557923CEF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_nw_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008605_nw_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_nw_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [547433, 3422482, 554036, 3430038], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6603], "eo:cloud_cover": 35, "proj:transform": [1, 0, 547433, 0, -1, 3430038, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0068 0103000020E61000000100000005000000CDCCCCCCCC8F55C0D2C2651536EF3E40AE2EA704C48F55C0F9F5436CB0003F4004AF963B339455C04B5645B8C9003F40B2F336363B9455C0E333D93F4FEF3E40CDCCCCCCCC8F55C0D2C2651536EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_ne_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008606_ne_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_ne_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [565333, 3422577, 571948, 3430144], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7567, 6615], "eo:cloud_cover": 22, "proj:transform": [1, 0, 565333, 0, -1, 3430144, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0069 0103000020E61000000100000005000000B58AFED0CC9355C08AAE0B3F38EF3E407E9065C1C49355C0B806B64AB0003F408C648F50339855C0B2834A5CC7003F406B47718E3A9855C0E333D93F4FEF3E40B58AFED0CC9355C08AAE0B3F38EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_nw_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008606_nw_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_nw_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [559366, 3422542, 565977, 3430105], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6611], "eo:cloud_cover": 97, "proj:transform": [1, 0, 559366, 0, -1, 3430105, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0070 0103000020E61000000100000005000000C7F5EFFACC8755C069A8514832EF3E40F0517FBDC28755C035289A07B0003F40AA7D3A1E338C55C0374F75C8CD003F4011C8258E3C8C55C02AE09EE74FEF3E40C7F5EFFACC8755C069A8514832EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_ne_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008607_ne_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_ne_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [577267, 3422658, 583889, 3430231], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6622], "eo:cloud_cover": 32, "proj:transform": [1, 0, 577267, 0, -1, 3430231, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0071 0103000020E61000000100000005000000C7F5EFFACC8B55C03FADA23F34EF3E40D8F50B76C38B55C058FE7C5BB0003F4063B7CF2A339055C0624A24D1CB003F40E15D2EE23B9055C048F949B54FEF3E40C7F5EFFACC8B55C03FADA23F34EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_nw_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008607_nw_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_nw_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [571300, 3422616, 577918, 3430186], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7570, 6618], "eo:cloud_cover": 68, "proj:transform": [1, 0, 571300, 0, -1, 3430186, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0072 0103000020E610000001000000050000007F2F8507CD7F55C01EA7E8482EEF3E40200A664CC17F55C035289A07B0003F40390A1005338455C0471E882CD2003F40401878EE3D8455C08FA50F5D50EF3E407F2F8507CD7F55C01EA7E8482EEF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_ne_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008608_ne_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_ne_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [589201, 3422752, 595831, 3430332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6630], "eo:cloud_cover": 30, "proj:transform": [1, 0, 589201, 0, -1, 3430332, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0073 0103000020E61000000100000005000000AFB321FFCC8355C011C5E40D30EF3E4008AEF204C28355C076172829B0003F40F243A511338855C0302AA913D0003F4028F04E3E3D8855C02AE09EE74FEF3E40AFB321FFCC8355C011C5E40D30EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_nw_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008608_nw_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_nw_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [583234, 3422703, 589860, 3430280], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6626], "eo:cloud_cover": 3, "proj:transform": [1, 0, 583234, 0, -1, 3430280, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0074 0103000020E610000001000000050000004AD40B3ECD6F55C09F008A9125EF3E4093E52494BE6F55C00C957F2DAF003F40B05417F0327455C07E18213CDA003F40F7E978CC407455C0D1949D7E50EF3E404AD40B3ECD6F55C09F008A9125EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_ne_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008502_ne_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_ne_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [613069, 3422979, 619715, 3430573], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7594, 6646], "eo:cloud_cover": 52, "proj:transform": [1, 0, 613069, 0, -1, 3430573, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0075 0103000020E6100000010000000500000050ABE80FCD7B55C0A8AAD0402CEF3E4009E23C9CC07B55C01152B7B3AF003F40698EACFC328055C07C2B1213D4003F403FFED2A23E8055C0728C648F50EF3E4050ABE80FCD7B55C0A8AAD0402CEF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_nw_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008501_nw_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_nw_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [595168, 3422804, 601802, 3430387], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7583, 6634], "eo:cloud_cover": 8, "proj:transform": [1, 0, 595168, 0, -1, 3430387, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0076 0103000020E6100000010000000500000008E57D1CCD7755C050C763062AEF3E40F1B913ECBF7755C01152B7B3AF003F4081D07AF8327C55C0D40E7F4DD6003F403FE42D573F7C55C0728C648F50EF3E4008E57D1CCD7755C050C763062AEF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_ne_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008501_ne_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_ne_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [601135, 3422859, 607773, 3430446], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7587, 6638], "eo:cloud_cover": 59, "proj:transform": [1, 0, 601135, 0, -1, 3430446, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0077 0103000020E61000000100000005000000BB61DBA2CCAB55C09545611745EF3E40933655F7C8AB55C0FFB27BF2B0003F40B6114F7633B055C0CAC2D7D7BA003F406DE34F5436B055C0C05DF6EB4EEF3E40BB61DBA2CCAB55C09545611745EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_nw_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008603_nw_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_nw_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [523566, 3422402, 530153, 3429944], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7542, 6587], "eo:cloud_cover": 64, "proj:transform": [1, 0, 523566, 0, -1, 3429944, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0078 0103000020E61000000100000005000000EBE5779ACCA755C0DE59BBED42EF3E40F3583332C8A755C082919735B1003F40459E245D33AC55C06495D233BD003F40CCD1E3F736AC55C06155BDFC4EEF3E40EBE5779ACCA755C0DE59BBED42EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_ne_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008603_ne_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_ne_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [529533, 3422417, 536124, 3429963], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7546, 6591], "eo:cloud_cover": 21, "proj:transform": [1, 0, 529533, 0, -1, 3429963, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0079 0103000020E61000000100000005000000A305685BCD6355C0FC1BB4571FEF3E4094331477BC6355C0C4E8B985AE003F406F6589CE326855C0E10D6954E0003F40252026E1426855C0365A0EF450EF3E40A305685BCD6355C0FC1BB4571FEF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_nw_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008504_nw_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_nw_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [630971, 3423185, 637629, 3430789], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7604, 6658], "eo:cloud_cover": 65, "proj:transform": [1, 0, 630971, 0, -1, 3430789, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0080 0103000020E61000000100000005000000EACBD24ECD5F55C0A9F57EA31DEF3E400B98C0ADBB5F55C024F1F274AE003F402E76FBAC326455C098F90E7EE2003F40B492567C436455C0DC0E0D8B51EF3E40EACBD24ECD5F55C0A9F57EA31DEF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_ne_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008504_ne_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_ne_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [636939, 3423261, 643601, 3430868], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7607, 6662], "eo:cloud_cover": 95, "proj:transform": [1, 0, 636939, 0, -1, 3430868, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0081 0103000020E61000000100000005000000D960E124CD6B55C04DDA54DD23EF3E4052103CBEBD6B55C00C957F2DAF003F409FE925C6327055C0D7FB8D76DC003F40CD22145B417055C077499C1551EF3E40D960E124CD6B55C04DDA54DD23EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_nw_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008503_nw_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_nw_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [619037, 3423045, 625687, 3430642], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7597, 6650], "eo:cloud_cover": 39, "proj:transform": [1, 0, 619037, 0, -1, 3430642, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0082 0103000020E610000001000000050000004AD40B3ECD6755C07218CC5F21EF3E40F321A81ABD6755C089B663EAAE003F4087A757CA326C55C04DF8A57EDE003F408542041C426C55C0F46A80D250EF3E404AD40B3ECD6755C07218CC5F21EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_ne_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008503_ne_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_ne_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [625004, 3423113, 631658, 3430714], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7601, 6654], "eo:cloud_cover": 26, "proj:transform": [1, 0, 625004, 0, -1, 3430714, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0001 0103000020E6100000010000000500000044FD2E6CCD5755C0174850FC18EF3E40C405A051BA5755C0E2016553AE003F408D7E349C325C55C007D15AD1E6003F409B1C3EE9445C55C09B1F7F6951EF3E4044FD2E6CCD5755C0174850FC18EF3E40 pgstac-test-collection 2011-08-25 00:00:00+00 2011-08-25 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_ne_16_1_20110825.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008505_ne_16_1_20110825.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_ne_16_1_20110825.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [648874, 3423421, 655544, 3431036], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7615, 6670], "eo:cloud_cover": 89, "proj:transform": [1, 0, 648874, 0, -1, 3431036, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0002 0103000020E610000001000000050000002CBB6070CD5B55C0CE33F6251BEF3E407D259012BB5B55C0A112D731AE003F40E6AF90B9326055C06DFE5F75E4003F403C2EAA45446055C05930F14751EF3E402CBB6070CD5B55C0CE33F6251BEF3E40 pgstac-test-collection 2011-08-25 00:00:00+00 2011-08-25 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008505_nw_16_1_20110825.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [642906, 3423339, 649572, 3430950], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7611, 6666], "eo:cloud_cover": 33, "proj:transform": [1, 0, 642906, 0, -1, 3430950, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0083 0103000020E61000000100000005000000C11E1329CD7355C015FDA19927EF3E40DA91EA3BBF7355C0D0622992AF003F40991249F4327855C0EB025E66D8003F402788BA0F407855C0EFAD484C50EF3E40C11E1329CD7355C015FDA19927EF3E40 pgstac-test-collection 2011-08-15 00:00:00+00 2011-08-15 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_nw_16_1_20110815.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008502_nw_16_1_20110815.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_nw_16_1_20110815.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [607102, 3422917, 613744, 3430508], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7591, 6642], "eo:cloud_cover": 26, "proj:transform": [1, 0, 607102, 0, -1, 3430508, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0086 0103000020E6100000010000000500000078D32D3BC4EF55C0070ABC934FDF3E4026FE28EACCEF55C0C1525DC0CBF03E40F98557923CF455C0B806B64AB0F03E40DA01D71533F455C09EB5DB2E34DF3E4078D32D3BC4EF55C0070ABC934FDF3E40 pgstac-test-collection 2011-08-01 00:00:00+00 2011-08-01 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_se_16_1_20110801.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_se_16_1_20110801.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_se_16_1_20110801.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2011", "proj:bbox": [422031, 3415689, 428653, 3423259], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7570, 6622], "eo:cloud_cover": 40, "proj:transform": [1, 0, 422031, 0, -1, 3423259, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0087 0103000020E610000001000000050000004E0CC9C9C4EB55C0E333D93F4FEF3E40FC5069C4CCEB55C04B5645B8C9003F4052D158FB3BF055C0F9F5436CB0003F403333333333F055C0D2C2651536EF3E404E0CC9C9C4EB55C0E333D93F4FEF3E40 pgstac-test-collection 2011-08-01 00:00:00+00 2011-08-01 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_nw_16_1_20110801.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_nw_16_1_20110801.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_nw_16_1_20110801.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2011", "proj:bbox": [428052, 3422577, 434667, 3430144], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7567, 6615], "eo:cloud_cover": 36, "proj:transform": [1, 0, 428052, 0, -1, 3430144, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0088 0103000020E61000000100000005000000D7C1C1DEC4EB55C048F949B54FDF3E409D4830D5CCEB55C00A67B796C9F03E40292499D53BF055C0B806B64AB0F03E40F243A51133F055C03788D68A36DF3E40D7C1C1DEC4EB55C048F949B54FDF3E40 pgstac-test-collection 2011-08-01 00:00:00+00 2011-08-01 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_sw_16_1_20110801.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_sw_16_1_20110801.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_sw_16_1_20110801.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2011", "proj:bbox": [428006, 3415651, 434624, 3423217], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6618], "eo:cloud_cover": 23, "proj:transform": [1, 0, 428006, 0, -1, 3423217, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0089 0103000020E61000000100000005000000FBB1497EC4EF55C0252367614F8F3E4038691A14CDEF55C01B9E5E29CBA03E405E656D533CF455C0F3380CE6AFA03E40B05417F032F455C09EB5DB2E348F3E40FBB1497EC4EF55C0252367614F8F3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_ne_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_ne_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_ne_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421780, 3381056, 428421, 3388625], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6641], "eo:cloud_cover": 36, "proj:transform": [1, 0, 421780, 0, -1, 3388625, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0090 0103000020E61000000100000005000000BF99982EC4EF55C0CBD765F84FCF3E40850662D9CCEF55C0DF6B088ECBE03E40A054FB743CF455C076172829B0E03E40698EACFC32F455C06283859334CF3E40BF99982EC4EF55C0CBD765F84FCF3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_ne_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_ne_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_ne_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421981, 3408763, 428607, 3416332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6626], "eo:cloud_cover": 84, "proj:transform": [1, 0, 421981, 0, -1, 3416332, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0091 0103000020E6100000010000000500000001892650C4EF55C048F949B54F9F3E4026FE28EACCEF55C03F74417DCBB03E40179F02603CF455C076172829B0B03E4081D07AF832F455C0C18BBE82349F3E4001892650C4EF55C048F949B54F9F3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_se_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_se_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_se_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421830, 3387983, 428468, 3395552], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6638], "eo:cloud_cover": 31, "proj:transform": [1, 0, 421830, 0, -1, 3395552, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0092 0103000020E61000000100000005000000E9465854C4EF55C048F949B54FBF3E40DF37BEF6CCEF55C0FD84B35BCBD03E40404CC2853CF455C09430D3F6AFD03E40DA01D71533F455C0809C306134BF3E40E9465854C4EF55C048F949B54FBF3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_se_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_se_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_se_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421930, 3401836, 428560, 3409405], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6630], "eo:cloud_cover": 24, "proj:transform": [1, 0, 421930, 0, -1, 3409405, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0093 0103000020E61000000100000005000000FA97A432C5EB55C0252367614F8F3E4067EDB60BCDEB55C0C9772975C9A03E405F7F129F3BF055C0F9F5436CB0A03E4081D07AF832F055C055A18158368F3E40FA97A432C5EB55C0252367614F8F3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_nw_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_nw_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_nw_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427774, 3381018, 434411, 3388584], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6637], "eo:cloud_cover": 61, "proj:transform": [1, 0, 427774, 0, -1, 3388584, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0094 0103000020E6100000010000000500000030F31DFCC4EB55C0C51A2E724FCF3E4026FE28EACCEB55C028806264C9E03E40F99FFCDD3BF055C076172829B0E03E40AA7D3A1E33F055C055A1815836CF3E4030F31DFCC4EB55C0C51A2E724FCF3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_nw_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_nw_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_nw_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427959, 3408724, 434581, 3416290], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6622], "eo:cloud_cover": 29, "proj:transform": [1, 0, 427959, 0, -1, 3416290, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0095 0103000020E6100000010000000500000078B988EFC4EB55C08AE8D7D64FBF3E40850662D9CCEB55C0EC4D0CC9C9D03E40B8B06EBC3BF055C03BE5D18DB0D03E40390A100533F055C0797764AC36BF3E4078B988EFC4EB55C08AE8D7D64FBF3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_sw_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_sw_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_sw_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427913, 3401798, 434539, 3409364], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6626], "eo:cloud_cover": 63, "proj:transform": [1, 0, 427913, 0, -1, 3409364, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0096 0103000020E61000000100000005000000B9A81611C5EB55C06612F5824FAF3E40DF37BEF6CCEB55C0696FF085C9C03E405F7F129F3BF055C058FE7C5BB0C03E40B05417F032F055C0F698486936AF3E40B9A81611C5EB55C06612F5824FAF3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_nw_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_nw_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_nw_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427867, 3394871, 434496, 3402437], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6629], "eo:cloud_cover": 26, "proj:transform": [1, 0, 427867, 0, -1, 3402437, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0097 0103000020E6100000010000000500000001892650C4EF55C0C51A2E724FAF3E400EBC5AEECCEF55C0C1525DC0CBC03E40B796C9703CF455C0F9F5436CB0C03E40390A100533F455C09EB5DB2E34AF3E4001892650C4EF55C0C51A2E724FAF3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_ne_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_ne_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_ne_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421880, 3394909, 428514, 3402479], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7570, 6634], "eo:cloud_cover": 1, "proj:transform": [1, 0, 421880, 0, -1, 3402479, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0098 0103000020E61000000100000005000000E92CB308C5EB55C0E9F010C64F9F3E4026FE28EACCEB55C046990D32C9B03E40E7340BB43BF055C0D61F6118B0B03E4021C8410933F055C01A6F2BBD369F3E40E92CB308C5EB55C0E9F010C64F9F3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_sw_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_sw_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_sw_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427820, 3387945, 434454, 3395510], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7565, 6634], "eo:cloud_cover": 73, "proj:transform": [1, 0, 427820, 0, -1, 3395510, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0099 0103000020E61000000100000005000000FA97A432C5EB55C0070ABC934F7F3E407F2F8507CDEB55C0A5A14621C9903E40BE874B8E3BF055C035289A07B0903E40B05417F032F055C0D87F9D9B367F3E40FA97A432C5EB55C0070ABC934F7F3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_sw_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_sw_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_sw_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427728, 3374092, 434369, 3381657], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7565, 6641], "eo:cloud_cover": 90, "proj:transform": [1, 0, 427728, 0, -1, 3381657, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0084 0103000020E610000001000000050000003E26529ACD4F55C09014916115EF3E4036AD1402B94F55C01E34BBEEAD003F408D7E349C325455C094C151F2EA003F403BE0BA62465455C023BBD23252EF3E403E26529ACD4F55C09014916115EF3E40 pgstac-test-collection 2011-08-02 00:00:00+00 2011-08-02 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_ne_16_1_20110802.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_ne_16_1_20110802.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_ne_16_1_20110802.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-02T00:00:00Z", "naip:year": "2011", "proj:bbox": [660809, 3423596, 667487, 3431217], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7621, 6678], "eo:cloud_cover": 52, "proj:transform": [1, 0, 660809, 0, -1, 3431217, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0085 0103000020E610000001000000050000001FA2D11DC4EF55C048F949B54FEF3E409D4830D5CCEF55C0624A24D1CB003F40280AF4893CF455C058FE7C5BB0003F40390A100533F455C03FADA23F34EF3E401FA2D11DC4EF55C048F949B54FEF3E40 pgstac-test-collection 2011-08-01 00:00:00+00 2011-08-01 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_ne_16_1_20110801.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_ne_16_1_20110801.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_ne_16_1_20110801.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2011", "proj:bbox": [422082, 3422616, 428700, 3430186], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7570, 6618], "eo:cloud_cover": 3, "proj:transform": [1, 0, 422082, 0, -1, 3430186, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} pgstac-test-item-0100 0103000020E61000000100000005000000CB2DAD86C4EF55C0070ABC934F7F3E4038691A14CDEF55C09E7C7A6CCB903E40A62BD8463CF455C076172829B0903E40C896E5EB32F455C02194F771347F3E40CB2DAD86C4EF55C0070ABC934F7F3E40 pgstac-test-collection 2011-07-31 00:00:00+00 2011-07-31 00:00:00+00 {"assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_se_16_1_20110731.tif"}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_se_16_1_20110731.txt"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_se_16_1_20110731.200.jpg"}}, "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421730, 3374130, 428375, 3381699], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6645], "eo:cloud_cover": 50, "proj:transform": [1, 0, 421730, 0, -1, 3381699, 0, 0, 1]}, "stac_extensions": ["eo", "projection"]} ================================================ FILE: src/pgstac/tests/testdata/items_duplicate_ids.ndjson ================================================ {"id": "pgstac-test-item-duplicated", "bbox": [-87.816179, 30.496894, -87.74637, 30.565604], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_se_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_se_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_se_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.74637, 30.497308], [-87.746892, 30.565604], [-87.816179, 30.565188], [-87.815608, 30.496894], [-87.74637, 30.497308]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": 2013, "proj:bbox": [421730, 3374130, 428375, 3381699], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "zz", "proj:shape": [7569, 6645], "eo:cloud_cover": 50, "proj:transform": [1, 0, 421730, 0, -1, 3381699, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-duplicated", "bbox": [-87.816179, 30.496894, -87.74637, 30.565604], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_se_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_se_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_se_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.74637, 30.497308], [-87.746892, 30.565604], [-87.816179, 30.565188], [-87.815608, 30.496894], [-87.74637, 30.497308]]]}, "collection": "pgstac-test-collection2", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": 2013, "proj:bbox": [421730, 3374130, 428375, 3381699], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "zz", "proj:shape": [7569, 6645], "eo:cloud_cover": 50, "proj:transform": [1, 0, 421730, 0, -1, 3381699, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} ================================================ FILE: src/pgstac/tests/testdata/items_for_delsert.ndjson ================================================ {"id": "pgstac-test-item-9993", "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-9994", "bbox": [-87.190775, 30.934712, -87.121772, 31.002794], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_ne_16_1_20110824.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008707_ne_16_1_20110824.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_ne_16_1_20110824.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.121772, 30.934795], [-87.121858, 31.002794], [-87.190775, 31.002711], [-87.190639, 30.934712], [-87.121772, 30.934795]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [481788, 3422382, 488367, 3429918], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7536, 6579], "eo:cloud_cover": 23, "proj:transform": [1, 0, 481788, 0, -1, 3429918, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-9995", "bbox": [-87.128238, 30.934744, -87.059311, 31.002757], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_nw_16_1_20110824.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008708_nw_16_1_20110824.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_nw_16_1_20110824.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.059311, 30.934794], [-87.059353, 31.002757], [-87.128238, 31.002707], [-87.128147, 30.934744], [-87.059311, 30.934794]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [487758, 3422377, 494334, 3429909], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7532, 6576], "eo:cloud_cover": 3, "proj:transform": [1, 0, 487758, 0, -1, 3429909, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0006", "bbox": [-87.253322, 30.934677, -87.184223, 31.00282], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_nw_16_1_20110824.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008707_nw_16_1_20110824.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_nw_16_1_20110824.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.184223, 30.934794], [-87.184354, 31.00282], [-87.253322, 31.002703], [-87.253142, 30.934677], [-87.184223, 30.934794]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [475817, 3422390, 482401, 3429929], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7539, 6584], "eo:cloud_cover": 100, "proj:transform": [1, 0, 475817, 0, -1, 3429929, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0007", "bbox": [-87.440935, 30.622081, -87.371617, 30.690415], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_se_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_se_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_se_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.371617, 30.622297], [-87.371878, 30.690415], [-87.440935, 30.690199], [-87.440626, 30.622081], [-87.371617, 30.622297]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [457770, 3387803, 464384, 3395352], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7549, 6614], "eo:cloud_cover": 59, "proj:transform": [1, 0, 457770, 0, -1, 3395352, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0008", "bbox": [-87.503478, 30.622053, -87.434074, 30.690447], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_sw_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_sw_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_sw_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.434074, 30.622302], [-87.434379, 30.690447], [-87.503478, 30.690198], [-87.503124, 30.622053], [-87.434074, 30.622302]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 999, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [451780, 3387825, 458398, 3395377], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7552, 6618], "eo:cloud_cover": 64, "proj:transform": [1, 0, 451780, 0, -1, 3395377, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} ================================================ FILE: src/pgstac/tests/testdata/items_private.ndjson ================================================ {"id": "pgstac-test-item-0003", "private": {"created_by": "stac-task"}, "bbox": [-85.379245, 30.933949, -85.308201, 31.003555], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.309412, 30.933949], [-85.308201, 31.002658], [-85.378084, 31.003555], [-85.379245, 30.934843], [-85.309412, 30.933949]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [654842, 3423507, 661516, 3431125], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7618, 6674], "eo:cloud_cover": 28, "proj:transform": [1, 0, 654842, 0, -1, 3431125, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0004", "private": {"created_by": "stac-task"}, "bbox": [-87.190775, 30.934712, -87.121772, 31.002794], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_ne_16_1_20110824.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008707_ne_16_1_20110824.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_ne_16_1_20110824.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.121772, 30.934795], [-87.121858, 31.002794], [-87.190775, 31.002711], [-87.190639, 30.934712], [-87.121772, 30.934795]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [481788, 3422382, 488367, 3429918], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7536, 6579], "eo:cloud_cover": 23, "proj:transform": [1, 0, 481788, 0, -1, 3429918, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0005", "private": {"created_by": "stac-task"}, "bbox": [-87.128238, 30.934744, -87.059311, 31.002757], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_nw_16_1_20110824.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008708_nw_16_1_20110824.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_nw_16_1_20110824.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.059311, 30.934794], [-87.059353, 31.002757], [-87.128238, 31.002707], [-87.128147, 30.934744], [-87.059311, 30.934794]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [487758, 3422377, 494334, 3429909], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7532, 6576], "eo:cloud_cover": 3, "proj:transform": [1, 0, 487758, 0, -1, 3429909, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0006", "private": {"created_by": "stac-task"}, "bbox": [-87.253322, 30.934677, -87.184223, 31.00282], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_nw_16_1_20110824.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008707_nw_16_1_20110824.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008707_nw_16_1_20110824.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.184223, 30.934794], [-87.184354, 31.00282], [-87.253322, 31.002703], [-87.253142, 30.934677], [-87.184223, 30.934794]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-24T00:00:00Z", "naip:year": "2011", "proj:bbox": [475817, 3422390, 482401, 3429929], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7539, 6584], "eo:cloud_cover": 100, "proj:transform": [1, 0, 475817, 0, -1, 3429929, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0007", "private": {"created_by": "stac-task"}, "bbox": [-87.440935, 30.622081, -87.371617, 30.690415], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_se_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_se_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_se_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.371617, 30.622297], [-87.371878, 30.690415], [-87.440935, 30.690199], [-87.440626, 30.622081], [-87.371617, 30.622297]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [457770, 3387803, 464384, 3395352], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7549, 6614], "eo:cloud_cover": 59, "proj:transform": [1, 0, 457770, 0, -1, 3395352, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0008", "private": {"created_by": "stac-task"}, "bbox": [-87.503478, 30.622053, -87.434074, 30.690447], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_sw_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_sw_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_sw_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.434074, 30.622302], [-87.434379, 30.690447], [-87.503478, 30.690198], [-87.503124, 30.622053], [-87.434074, 30.622302]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [451780, 3387825, 458398, 3395377], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7552, 6618], "eo:cloud_cover": 64, "proj:transform": [1, 0, 451780, 0, -1, 3395377, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0009", "private": {"created_by": "stac-task"}, "bbox": [-87.503478, 30.68455, -87.434072, 30.752944], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_nw_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_nw_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_nw_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.434072, 30.684799], [-87.434377, 30.752944], [-87.503478, 30.752694], [-87.503125, 30.68455], [-87.434072, 30.684799]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [451811, 3394751, 458425, 3402303], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7552, 6614], "eo:cloud_cover": 61, "proj:transform": [1, 0, 451811, 0, -1, 3402303, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0010", "private": {"created_by": "stac-task"}, "bbox": [-87.440937, 30.684579, -87.371606, 30.752912], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_ne_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008721_ne_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008721_ne_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.371606, 30.684794], [-87.371867, 30.752912], [-87.440937, 30.752696], [-87.440628, 30.684579], [-87.371606, 30.684794]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [457797, 3394729, 464408, 3402278], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7549, 6611], "eo:cloud_cover": 31, "proj:transform": [1, 0, 457797, 0, -1, 3402278, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0011", "private": {"created_by": "stac-task"}, "bbox": [-87.315859, 30.934648, -87.246684, 31.002851], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_ne_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008706_ne_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_ne_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.246684, 30.934798], [-87.246859, 31.002851], [-87.315859, 31.0027], [-87.315635, 30.934648], [-87.246684, 30.934798]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [469847, 3422402, 476434, 3429944], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7542, 6587], "eo:cloud_cover": 41, "proj:transform": [1, 0, 469847, 0, -1, 3429944, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0012", "private": {"created_by": "stac-task"}, "bbox": [-87.440942, 30.93458, -87.371596, 31.002921], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_ne_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008705_ne_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_ne_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.371596, 30.934797], [-87.37186, 31.002921], [-87.440942, 31.002704], [-87.440629, 30.93458], [-87.371596, 30.934797]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [457906, 3422435, 464501, 3429985], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7550, 6595], "eo:cloud_cover": 4, "proj:transform": [1, 0, 457906, 0, -1, 3429985, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0013", "private": {"created_by": "stac-task"}, "bbox": [-87.378406, 30.934615, -87.309145, 31.002887], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_nw_16_1_20110817.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008706_nw_16_1_20110817.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008706_nw_16_1_20110817.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.309145, 30.934799], [-87.309365, 31.002887], [-87.378406, 31.002704], [-87.378137, 30.934615], [-87.309145, 30.934799]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-17T00:00:00Z", "naip:year": "2011", "proj:bbox": [463876, 3422417, 470467, 3429963], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7546, 6591], "eo:cloud_cover": 2, "proj:transform": [1, 0, 463876, 0, -1, 3429963, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0014", "private": {"created_by": "stac-task"}, "bbox": [-87.628547, 30.496984, -87.558991, 30.565507], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558991, 30.497299], [-87.559382, 30.565507], [-87.628547, 30.565192], [-87.628108, 30.496984], [-87.558991, 30.497299]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439724, 3374025, 446357, 3381584], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6633], "eo:cloud_cover": 17, "proj:transform": [1, 0, 439724, 0, -1, 3381584, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0015", "private": {"created_by": "stac-task"}, "bbox": [-86.940689, 30.934744, -86.871762, 31.002757], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008601_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.871853, 30.934744], [-86.871762, 31.002707], [-86.940647, 31.002757], [-86.940689, 30.934794], [-86.871853, 30.934744]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [505666, 3422377, 512242, 3429909], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7532, 6576], "eo:cloud_cover": 54, "proj:transform": [1, 0, 505666, 0, -1, 3429909, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0016", "private": {"created_by": "stac-task"}, "bbox": [-87.003143, 30.934773, -86.93431, 31.002726], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008601_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008601_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.934356, 30.934773], [-86.93431, 31.002709], [-87.003143, 31.002726], [-87.00314, 30.934789], [-86.934356, 30.934773]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [499700, 3422375, 506271, 3429904], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7529, 6571], "eo:cloud_cover": 13, "proj:transform": [1, 0, 499700, 0, -1, 3429904, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0017", "private": {"created_by": "stac-task"}, "bbox": [-86.815777, 30.934677, -86.746678, 31.00282], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008602_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.746858, 30.934677], [-86.746678, 31.002703], [-86.815646, 31.00282], [-86.815777, 30.934794], [-86.746858, 30.934677]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [517599, 3422390, 524183, 3429929], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7539, 6584], "eo:cloud_cover": 59, "proj:transform": [1, 0, 517599, 0, -1, 3429929, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0018", "private": {"created_by": "stac-task"}, "bbox": [-86.878228, 30.934712, -86.809225, 31.002794], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008602_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008602_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.809361, 30.934712], [-86.809225, 31.002711], [-86.878142, 31.002794], [-86.878228, 30.934795], [-86.809361, 30.934712]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [511633, 3422382, 518212, 3429918], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7536, 6579], "eo:cloud_cover": 29, "proj:transform": [1, 0, 511633, 0, -1, 3429918, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0019", "private": {"created_by": "stac-task"}, "bbox": [-87.566035, 30.934518, -87.496517, 31.002979], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008704_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496517, 30.934802], [-87.49687, 31.002979], [-87.566035, 31.002694], [-87.565633, 30.934518], [-87.496517, 30.934802]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445964, 3422482, 452567, 3430038], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6603], "eo:cloud_cover": 52, "proj:transform": [1, 0, 445964, 0, -1, 3430038, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0020", "private": {"created_by": "stac-task"}, "bbox": [-87.62857, 30.934491, -87.558977, 31.003012], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008704_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558977, 30.934808], [-87.559374, 31.003012], [-87.62857, 31.002694], [-87.628123, 30.934491], [-87.558977, 30.934808]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439994, 3422511, 446600, 3430070], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6606], "eo:cloud_cover": 39, "proj:transform": [1, 0, 439994, 0, -1, 3430070, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0021", "private": {"created_by": "stac-task"}, "bbox": [-87.628569, 30.871988, -87.55898, 30.940509], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008704_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008704_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.55898, 30.872305], [-87.559377, 30.940509], [-87.628569, 30.940191], [-87.628123, 30.871988], [-87.55898, 30.872305]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439955, 3415584, 446565, 3423143], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6610], "eo:cloud_cover": 29, "proj:transform": [1, 0, 439955, 0, -1, 3423143, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0022", "private": {"created_by": "stac-task"}, "bbox": [-87.503488, 30.93455, -87.434056, 31.002952], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008705_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008705_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.434056, 30.934801], [-87.434365, 31.002952], [-87.503488, 31.002701], [-87.503131, 30.93455], [-87.434056, 30.934801]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [451935, 3422457, 458534, 3430010], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7553, 6599], "eo:cloud_cover": 84, "proj:transform": [1, 0, 451935, 0, -1, 3430010, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0023", "private": {"created_by": "stac-task"}, "bbox": [-87.06569, 30.934773, -86.996857, 31.002726], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008708_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008708_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.99686, 30.934789], [-86.996857, 31.002726], [-87.06569, 31.002709], [-87.065644, 30.934773], [-86.99686, 30.934789]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [493729, 3422375, 500300, 3429904], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7529, 6571], "eo:cloud_cover": 56, "proj:transform": [1, 0, 493729, 0, -1, 3429904, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0024", "private": {"created_by": "stac-task"}, "bbox": [-87.941273, 30.809325, -87.871271, 30.878173], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871271, 30.80981], [-87.871889, 30.878173], [-87.941273, 30.877687], [-87.940605, 30.809325], [-87.871271, 30.80981]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [410024, 3408849, 416657, 3416426], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6633], "eo:cloud_cover": 58, "proj:transform": [1, 0, 410024, 0, -1, 3416426, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0025", "private": {"created_by": "stac-task"}, "bbox": [-88.003818, 30.809299, -87.933721, 30.878206], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.933721, 30.809817], [-87.934384, 30.878206], [-88.003818, 30.877686], [-88.003106, 30.809299], [-87.933721, 30.809817]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [404045, 3408898, 410683, 3416478], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6638], "eo:cloud_cover": 65, "proj:transform": [1, 0, 404045, 0, -1, 3416478, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0026", "private": {"created_by": "stac-task"}, "bbox": [-87.941269, 30.746832, -87.871272, 30.815671], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871272, 30.747317], [-87.871889, 30.815671], [-87.941269, 30.815185], [-87.940603, 30.746832], [-87.871272, 30.747317]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409966, 3401923, 416603, 3409499], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6637], "eo:cloud_cover": 52, "proj:transform": [1, 0, 409966, 0, -1, 3409499, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0027", "private": {"created_by": "stac-task"}, "bbox": [-88.003816, 30.746797, -87.933724, 30.815704], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008709_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008709_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.933724, 30.747315], [-87.934385, 30.815704], [-88.003816, 30.815185], [-88.003106, 30.746797], [-87.933724, 30.747315]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [403983, 3401971, 410625, 3409551], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6642], "eo:cloud_cover": 43, "proj:transform": [1, 0, 403983, 0, -1, 3409551, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0028", "private": {"created_by": "stac-task"}, "bbox": [-87.878737, 30.809358, -87.80881, 30.878137], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.80881, 30.809809], [-87.809384, 30.878137], [-87.878737, 30.877684], [-87.878114, 30.809358], [-87.80881, 30.809809]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [416002, 3408804, 422632, 3416377], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6630], "eo:cloud_cover": 46, "proj:transform": [1, 0, 416002, 0, -1, 3416377, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0029", "private": {"created_by": "stac-task"}, "bbox": [-87.878732, 30.746864, -87.80881, 30.815634], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.80881, 30.747315], [-87.809382, 30.815634], [-87.878732, 30.815182], [-87.878111, 30.746864], [-87.80881, 30.747315]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415948, 3401878, 422582, 3409450], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7572, 6634], "eo:cloud_cover": 42, "proj:transform": [1, 0, 415948, 0, -1, 3409450, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0030", "private": {"created_by": "stac-task"}, "bbox": [-87.691106, 30.809455, -87.621436, 30.878045], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621436, 30.809805], [-87.621876, 30.878045], [-87.691106, 30.877694], [-87.690617, 30.809455], [-87.621436, 30.809805]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433938, 3408689, 440556, 3416252], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6618], "eo:cloud_cover": 16, "proj:transform": [1, 0, 433938, 0, -1, 3416252, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0031", "private": {"created_by": "stac-task"}, "bbox": [-87.691108, 30.74696, -87.621442, 30.815542], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621442, 30.74731], [-87.62188, 30.815542], [-87.691108, 30.815191], [-87.69062, 30.74696], [-87.621442, 30.74731]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433895, 3401763, 440517, 3409325], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6622], "eo:cloud_cover": 16, "proj:transform": [1, 0, 433895, 0, -1, 3409325, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0032", "private": {"created_by": "stac-task"}, "bbox": [-87.628569, 30.809484, -87.558973, 30.878015], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008712_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558973, 30.809801], [-87.559369, 30.878015], [-87.628569, 30.877697], [-87.628124, 30.809484], [-87.558973, 30.809801]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439916, 3408657, 446531, 3416217], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7560, 6615], "eo:cloud_cover": 7, "proj:transform": [1, 0, 439916, 0, -1, 3416217, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0033", "private": {"created_by": "stac-task"}, "bbox": [-87.566019, 30.747015, -87.496524, 30.815477], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008712_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496524, 30.747298], [-87.496874, 30.815477], [-87.566019, 30.815193], [-87.56562, 30.747015], [-87.496524, 30.747298]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445860, 3401702, 452474, 3409258], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6614], "eo:cloud_cover": 49, "proj:transform": [1, 0, 445860, 0, -1, 3409258, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0034", "private": {"created_by": "stac-task"}, "bbox": [-87.628559, 30.746989, -87.558978, 30.815511], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008712_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008712_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558978, 30.747305], [-87.559372, 30.815511], [-87.628559, 30.815194], [-87.628115, 30.746989], [-87.558978, 30.747305]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439878, 3401731, 446496, 3409290], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6618], "eo:cloud_cover": 3, "proj:transform": [1, 0, 439878, 0, -1, 3409290, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0035", "private": {"created_by": "stac-task"}, "bbox": [-87.941266, 30.68433, -87.871274, 30.753168], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871274, 30.684813], [-87.871889, 30.753168], [-87.941266, 30.752684], [-87.940602, 30.68433], [-87.871274, 30.684813]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409908, 3394996, 416549, 3402572], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6641], "eo:cloud_cover": 33, "proj:transform": [1, 0, 409908, 0, -1, 3402572, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0036", "private": {"created_by": "stac-task"}, "bbox": [-88.003814, 30.684295, -87.933727, 30.753202], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.933727, 30.684813], [-87.934386, 30.753202], [-88.003814, 30.752684], [-88.003106, 30.684295], [-87.933727, 30.684813]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [403921, 3395044, 410567, 3402624], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6646], "eo:cloud_cover": 12, "proj:transform": [1, 0, 403921, 0, -1, 3402624, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0037", "private": {"created_by": "stac-task"}, "bbox": [-87.941265, 30.621827, -87.871277, 30.690665], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871277, 30.62231], [-87.871891, 30.690665], [-87.941265, 30.690181], [-87.940603, 30.621827], [-87.871277, 30.62231]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409850, 3388069, 416495, 3395645], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6645], "eo:cloud_cover": 54, "proj:transform": [1, 0, 409850, 0, -1, 3395645, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0038", "private": {"created_by": "stac-task"}, "bbox": [-88.003804, 30.621802, -87.933732, 30.6907], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008717_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008717_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.933732, 30.622318], [-87.934389, 30.6907], [-88.003804, 30.690182], [-88.003098, 30.621802], [-87.933732, 30.622318]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [403860, 3388118, 410509, 3395697], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7579, 6649], "eo:cloud_cover": 77, "proj:transform": [1, 0, 403860, 0, -1, 3395697, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0039", "private": {"created_by": "stac-task"}, "bbox": [-87.878728, 30.684361, -87.808821, 30.75314], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808821, 30.684811], [-87.809392, 30.75314], [-87.878728, 30.752689], [-87.878109, 30.684361], [-87.808821, 30.684811]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415894, 3394951, 422531, 3402524], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6637], "eo:cloud_cover": 12, "proj:transform": [1, 0, 415894, 0, -1, 3402524, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0040", "private": {"created_by": "stac-task"}, "bbox": [-87.878725, 30.621858, -87.808823, 30.690637], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808823, 30.622307], [-87.809392, 30.690637], [-87.878725, 30.690186], [-87.878107, 30.621858], [-87.808823, 30.622307]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415840, 3388024, 422481, 3395597], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6641], "eo:cloud_cover": 10, "proj:transform": [1, 0, 415840, 0, -1, 3395597, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0041", "private": {"created_by": "stac-task"}, "bbox": [-87.6911, 30.684456, -87.621438, 30.753047], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621438, 30.684805], [-87.621875, 30.753047], [-87.6911, 30.752696], [-87.690613, 30.684456], [-87.621438, 30.684805]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433853, 3394836, 440479, 3402399], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6626], "eo:cloud_cover": 3, "proj:transform": [1, 0, 433853, 0, -1, 3402399, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0042", "private": {"created_by": "stac-task"}, "bbox": [-87.691103, 30.62196, -87.621445, 30.690542], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621445, 30.622309], [-87.621882, 30.690542], [-87.691103, 30.690192], [-87.690618, 30.62196], [-87.621445, 30.622309]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433810, 3387910, 440440, 3395472], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6630], "eo:cloud_cover": 99, "proj:transform": [1, 0, 433810, 0, -1, 3395472, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0043", "private": {"created_by": "stac-task"}, "bbox": [-87.566019, 30.684519, -87.496527, 30.752981], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496527, 30.684801], [-87.496877, 30.752981], [-87.566019, 30.752698], [-87.565621, 30.684519], [-87.496527, 30.684801]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445825, 3394776, 452443, 3402332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6618], "eo:cloud_cover": 80, "proj:transform": [1, 0, 445825, 0, -1, 3402332, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0044", "private": {"created_by": "stac-task"}, "bbox": [-87.62856, 30.684484, -87.558983, 30.753015], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558983, 30.6848], [-87.559376, 30.753015], [-87.62856, 30.752699], [-87.628117, 30.684484], [-87.558983, 30.6848]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439839, 3394804, 446461, 3402364], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7560, 6622], "eo:cloud_cover": 30, "proj:transform": [1, 0, 439839, 0, -1, 3402364, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0045", "private": {"created_by": "stac-task"}, "bbox": [-87.56602, 30.622022, -87.496532, 30.690476], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496532, 30.622304], [-87.49688, 30.690476], [-87.56602, 30.690193], [-87.565623, 30.622022], [-87.496532, 30.622304]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445790, 3387850, 452412, 3395405], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7555, 6622], "eo:cloud_cover": 82, "proj:transform": [1, 0, 445790, 0, -1, 3395405, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0046", "private": {"created_by": "stac-task"}, "bbox": [-87.628562, 30.621988, -87.558988, 30.69051], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008720_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008720_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558988, 30.622303], [-87.559381, 30.69051], [-87.628562, 30.690194], [-87.62812, 30.621988], [-87.558988, 30.622303]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439800, 3387878, 446426, 3395437], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6626], "eo:cloud_cover": 16, "proj:transform": [1, 0, 439800, 0, -1, 3395437, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0047", "private": {"created_by": "stac-task"}, "bbox": [-87.941265, 30.559332, -87.871282, 30.628171], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008725_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871282, 30.559814], [-87.871893, 30.628171], [-87.941265, 30.627687], [-87.940604, 30.559332], [-87.871282, 30.559814]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409792, 3381143, 416441, 3388719], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6649], "eo:cloud_cover": 43, "proj:transform": [1, 0, 409792, 0, -1, 3388719, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0048", "private": {"created_by": "stac-task"}, "bbox": [-87.941255, 30.496828, -87.871287, 30.565666], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008725_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008725_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871287, 30.497309], [-87.871897, 30.565666], [-87.941255, 30.565183], [-87.940596, 30.496828], [-87.871287, 30.497309]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [409735, 3374216, 416387, 3381792], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7576, 6652], "eo:cloud_cover": 4, "proj:transform": [1, 0, 409735, 0, -1, 3381792, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0049", "private": {"created_by": "stac-task"}, "bbox": [-87.878723, 30.559362, -87.808825, 30.628132], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808825, 30.559811], [-87.809393, 30.628132], [-87.878723, 30.627682], [-87.878107, 30.559362], [-87.808825, 30.559811]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415786, 3381098, 422431, 3388670], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7572, 6645], "eo:cloud_cover": 9, "proj:transform": [1, 0, 415786, 0, -1, 3388670, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0050", "private": {"created_by": "stac-task"}, "bbox": [-87.878723, 30.496867, -87.808828, 30.565637], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808828, 30.497315], [-87.809395, 30.565637], [-87.878723, 30.565187], [-87.878108, 30.496867], [-87.808828, 30.497315]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [415732, 3374172, 422381, 3381744], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7572, 6649], "eo:cloud_cover": 84, "proj:transform": [1, 0, 415732, 0, -1, 3381744, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0051", "private": {"created_by": "stac-task"}, "bbox": [-87.691097, 30.559454, -87.621453, 30.628045], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621453, 30.559803], [-87.621889, 30.628045], [-87.691097, 30.627696], [-87.690613, 30.559454], [-87.621453, 30.559803]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433768, 3380983, 440401, 3388546], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6633], "eo:cloud_cover": 22, "proj:transform": [1, 0, 433768, 0, -1, 3388546, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0052", "private": {"created_by": "stac-task"}, "bbox": [-87.691091, 30.496957, -87.621451, 30.565539], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621451, 30.497305], [-87.621886, 30.565539], [-87.691091, 30.565191], [-87.690608, 30.496957], [-87.621451, 30.497305]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433726, 3374057, 440363, 3381619], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6637], "eo:cloud_cover": 12, "proj:transform": [1, 0, 433726, 0, -1, 3381619, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0053", "private": {"created_by": "stac-task"}, "bbox": [-87.56601, 30.559516, -87.496536, 30.627979], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496536, 30.559797], [-87.496884, 30.627979], [-87.56601, 30.627696], [-87.565614, 30.559516], [-87.496536, 30.559797]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445756, 3380923, 452381, 3388479], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6625], "eo:cloud_cover": 24, "proj:transform": [1, 0, 445756, 0, -1, 3388479, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0054", "private": {"created_by": "stac-task"}, "bbox": [-87.628554, 30.559491, -87.558995, 30.628014], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.558995, 30.559806], [-87.559386, 30.628014], [-87.628554, 30.627698], [-87.628114, 30.559491], [-87.558995, 30.559806]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [439762, 3380952, 446391, 3388511], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6629], "eo:cloud_cover": 5, "proj:transform": [1, 0, 439762, 0, -1, 3388511, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0055", "private": {"created_by": "stac-task"}, "bbox": [-87.566012, 30.497018, -87.496531, 30.565481], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008728_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008728_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.496531, 30.497299], [-87.496878, 30.565481], [-87.566012, 30.565199], [-87.565617, 30.497018], [-87.496531, 30.497299]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [445721, 3373997, 452351, 3381553], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6630], "eo:cloud_cover": 79, "proj:transform": [1, 0, 445721, 0, -1, 3381553, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0056", "private": {"created_by": "stac-task"}, "bbox": [-87.941283, 30.934327, -87.871262, 31.003175], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871262, 30.934813], [-87.871883, 31.003175], [-87.941283, 31.002688], [-87.940613, 30.934327], [-87.871262, 30.934813]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [410140, 3422703, 416766, 3430280], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6626], "eo:cloud_cover": 62, "proj:transform": [1, 0, 410140, 0, -1, 3430280, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0057", "private": {"created_by": "stac-task"}, "bbox": [-88.003827, 30.9343, -87.93372, 31.003207], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.93372, 30.93482], [-87.934386, 31.003207], [-88.003827, 31.002686], [-88.003111, 30.9343], [-87.93372, 30.93482]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [404169, 3422752, 410799, 3430332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6630], "eo:cloud_cover": 7, "proj:transform": [1, 0, 404169, 0, -1, 3430332, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0058", "private": {"created_by": "stac-task"}, "bbox": [-87.941277, 30.871827, -87.871261, 30.940674], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.871261, 30.872312], [-87.87188, 30.940674], [-87.941277, 30.940187], [-87.940609, 30.871827], [-87.871261, 30.872312]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [410082, 3415776, 416712, 3423353], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6630], "eo:cloud_cover": 53, "proj:transform": [1, 0, 410082, 0, -1, 3423353, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0059", "private": {"created_by": "stac-task"}, "bbox": [-88.003822, 30.8718, -87.93372, 30.940707], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008701_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008701_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.93372, 30.872319], [-87.934384, 30.940707], [-88.003822, 30.940186], [-88.003108, 30.8718], [-87.93372, 30.872319]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [404107, 3415825, 410741, 3423405], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6634], "eo:cloud_cover": 57, "proj:transform": [1, 0, 404107, 0, -1, 3423405, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0066", "private": {"created_by": "stac-task"}, "bbox": [-86.441023, 30.934491, -86.37143, 31.003012], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008605_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.371877, 30.934491], [-86.37143, 31.002694], [-86.440626, 31.003012], [-86.441023, 30.934808], [-86.371877, 30.934491]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [553400, 3422511, 560006, 3430070], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7559, 6606], "eo:cloud_cover": 86, "proj:transform": [1, 0, 553400, 0, -1, 3430070, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0060", "private": {"created_by": "stac-task"}, "bbox": [-87.878739, 30.934361, -87.808804, 31.00314], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_nw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_nw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_nw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808804, 30.934813], [-87.80938, 31.00314], [-87.878739, 31.002686], [-87.878114, 30.934361], [-87.808804, 30.934813]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [416111, 3422658, 422733, 3430231], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6622], "eo:cloud_cover": 70, "proj:transform": [1, 0, 416111, 0, -1, 3430231, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0061", "private": {"created_by": "stac-task"}, "bbox": [-87.878743, 30.87186, -87.808801, 30.940638], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_sw_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_sw_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_sw_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.808801, 30.872311], [-87.809376, 30.940638], [-87.878743, 30.940186], [-87.878119, 30.87186], [-87.808801, 30.872311]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [416056, 3415731, 422683, 3423304], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6627], "eo:cloud_cover": 94, "proj:transform": [1, 0, 416056, 0, -1, 3423304, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0062", "private": {"created_by": "stac-task"}, "bbox": [-87.691116, 30.934452, -87.621426, 31.003042], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_ne_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_ne_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_ne_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621426, 30.934803], [-87.621868, 31.003042], [-87.691116, 31.00269], [-87.690624, 30.934452], [-87.621426, 30.934803]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [434023, 3422542, 440634, 3430105], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6611], "eo:cloud_cover": 85, "proj:transform": [1, 0, 434023, 0, -1, 3430105, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0063", "private": {"created_by": "stac-task"}, "bbox": [-87.691116, 30.871958, -87.621431, 30.940539], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_se_16_1_20110816.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_se_16_1_20110816.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_se_16_1_20110816.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.621431, 30.872309], [-87.621872, 30.940539], [-87.691116, 30.940188], [-87.690626, 30.871958], [-87.621431, 30.872309]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-16T00:00:00Z", "naip:year": "2011", "proj:bbox": [433980, 3415616, 440595, 3423178], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7562, 6615], "eo:cloud_cover": 2, "proj:transform": [1, 0, 433980, 0, -1, 3423178, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0064", "private": {"created_by": "stac-task"}, "bbox": [-86.565944, 30.93455, -86.496512, 31.002952], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008604_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.496869, 30.93455], [-86.496512, 31.002701], [-86.565635, 31.002952], [-86.565944, 30.934801], [-86.496869, 30.93455]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [541466, 3422457, 548065, 3430010], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7553, 6599], "eo:cloud_cover": 40, "proj:transform": [1, 0, 541466, 0, -1, 3430010, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0065", "private": {"created_by": "stac-task"}, "bbox": [-86.628404, 30.93458, -86.559058, 31.002921], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008604_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008604_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.559371, 30.93458], [-86.559058, 31.002704], [-86.62814, 31.002921], [-86.628404, 30.934797], [-86.559371, 30.93458]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [535499, 3422435, 542094, 3429985], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7550, 6595], "eo:cloud_cover": 27, "proj:transform": [1, 0, 535499, 0, -1, 3429985, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0067", "private": {"created_by": "stac-task"}, "bbox": [-86.503483, 30.934518, -86.433965, 31.002979], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008605_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008605_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.434367, 30.934518], [-86.433965, 31.002694], [-86.50313, 31.002979], [-86.503483, 30.934802], [-86.434367, 30.934518]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [547433, 3422482, 554036, 3430038], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7556, 6603], "eo:cloud_cover": 35, "proj:transform": [1, 0, 547433, 0, -1, 3430038, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0068", "private": {"created_by": "stac-task"}, "bbox": [-86.316114, 30.934419, -86.246339, 31.003078], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008606_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.246875, 30.934419], [-86.246339, 31.002692], [-86.315627, 31.003078], [-86.316114, 30.934803], [-86.246875, 30.934419]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [565333, 3422577, 571948, 3430144], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7567, 6615], "eo:cloud_cover": 22, "proj:transform": [1, 0, 565333, 0, -1, 3430144, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0069", "private": {"created_by": "stac-task"}, "bbox": [-86.378574, 30.934452, -86.308884, 31.003042], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008606_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008606_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.309376, 30.934452], [-86.308884, 31.00269], [-86.378132, 31.003042], [-86.378574, 30.934803], [-86.309376, 30.934452]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [559366, 3422542, 565977, 3430105], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7563, 6611], "eo:cloud_cover": 97, "proj:transform": [1, 0, 559366, 0, -1, 3430105, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0070", "private": {"created_by": "stac-task"}, "bbox": [-86.191196, 30.934361, -86.121261, 31.00314], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008607_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.121886, 30.934361], [-86.121261, 31.002686], [-86.19062, 31.00314], [-86.191196, 30.934813], [-86.121886, 30.934361]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [577267, 3422658, 583889, 3430231], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7573, 6622], "eo:cloud_cover": 32, "proj:transform": [1, 0, 577267, 0, -1, 3430231, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0071", "private": {"created_by": "stac-task"}, "bbox": [-86.253655, 30.934391, -86.183805, 31.00311], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008607_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008607_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.184386, 30.934391], [-86.183805, 31.002691], [-86.253123, 31.00311], [-86.253655, 30.93481], [-86.184386, 30.934391]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [571300, 3422616, 577918, 3430186], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7570, 6618], "eo:cloud_cover": 68, "proj:transform": [1, 0, 571300, 0, -1, 3430186, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0072", "private": {"created_by": "stac-task"}, "bbox": [-86.06628, 30.9343, -85.996173, 31.003207], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008608_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.996889, 30.9343], [-85.996173, 31.002686], [-86.065614, 31.003207], [-86.06628, 30.93482], [-85.996889, 30.9343]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [589201, 3422752, 595831, 3430332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7580, 6630], "eo:cloud_cover": 30, "proj:transform": [1, 0, 589201, 0, -1, 3430332, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0073", "private": {"created_by": "stac-task"}, "bbox": [-86.128738, 30.934327, -86.058717, 31.003175], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008608_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008608_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.059387, 30.934327], [-86.058717, 31.002688], [-86.128117, 31.003175], [-86.128738, 30.934813], [-86.059387, 30.934327]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [583234, 3422703, 589860, 3430280], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7577, 6626], "eo:cloud_cover": 3, "proj:transform": [1, 0, 583234, 0, -1, 3430280, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0074", "private": {"created_by": "stac-task"}, "bbox": [-85.816455, 30.934167, -85.746007, 31.00333], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008502_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.746902, 30.934167], [-85.746007, 31.002673], [-85.815609, 31.00333], [-85.816455, 30.934822], [-85.746902, 30.934167]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [613069, 3422979, 619715, 3430573], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7594, 6646], "eo:cloud_cover": 52, "proj:transform": [1, 0, 613069, 0, -1, 3430573, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0075", "private": {"created_by": "stac-task"}, "bbox": [-86.003823, 30.934269, -85.933631, 31.003236], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008501_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.934391, 30.934269], [-85.933631, 31.002681], [-86.003112, 31.003236], [-86.003823, 30.934823], [-85.934391, 30.934269]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [595168, 3422804, 601802, 3430387], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7583, 6634], "eo:cloud_cover": 8, "proj:transform": [1, 0, 595168, 0, -1, 3430387, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0076", "private": {"created_by": "stac-task"}, "bbox": [-85.941366, 30.934235, -85.871089, 31.00327], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008501_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008501_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.871894, 30.934235], [-85.871089, 31.002681], [-85.940611, 31.00327], [-85.941366, 30.934823], [-85.871894, 30.934235]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [601135, 3422859, 607773, 3430446], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7587, 6638], "eo:cloud_cover": 59, "proj:transform": [1, 0, 601135, 0, -1, 3430446, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0077", "private": {"created_by": "stac-task"}, "bbox": [-86.753316, 30.934648, -86.684141, 31.002851], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008603_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.684365, 30.934648], [-86.684141, 31.0027], [-86.753141, 31.002851], [-86.753316, 30.934798], [-86.684365, 30.934648]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [523566, 3422402, 530153, 3429944], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7542, 6587], "eo:cloud_cover": 64, "proj:transform": [1, 0, 523566, 0, -1, 3429944, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0078", "private": {"created_by": "stac-task"}, "bbox": [-86.690855, 30.934615, -86.621594, 31.002887], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30086/m_3008603_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30086/m_3008603_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-86.621863, 30.934615], [-86.621594, 31.002704], [-86.690635, 31.002887], [-86.690855, 30.934799], [-86.621863, 30.934615]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [529533, 3422417, 536124, 3429963], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7546, 6591], "eo:cloud_cover": 21, "proj:transform": [1, 0, 529533, 0, -1, 3429963, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0079", "private": {"created_by": "stac-task"}, "bbox": [-85.629082, 30.934072, -85.558378, 31.003423], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008504_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.559409, 30.934072], [-85.558378, 31.002663], [-85.628101, 31.003423], [-85.629082, 30.934829], [-85.559409, 30.934072]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [630971, 3423185, 637629, 3430789], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7604, 6658], "eo:cloud_cover": 65, "proj:transform": [1, 0, 630971, 0, -1, 3430789, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0080", "private": {"created_by": "stac-task"}, "bbox": [-85.566619, 30.934046, -85.49583, 31.003456], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008504_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008504_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.496906, 30.934046], [-85.49583, 31.002662], [-85.565593, 31.003456], [-85.566619, 30.934838], [-85.496906, 30.934046]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [636939, 3423261, 643601, 3430868], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7607, 6662], "eo:cloud_cover": 95, "proj:transform": [1, 0, 636939, 0, -1, 3430868, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0081", "private": {"created_by": "stac-task"}, "bbox": [-85.753989, 30.934141, -85.683456, 31.003364], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008503_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.684396, 30.934141], [-85.683456, 31.002673], [-85.753099, 31.003364], [-85.753989, 30.934831], [-85.684396, 30.934141]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [619037, 3423045, 625687, 3430642], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7597, 6650], "eo:cloud_cover": 39, "proj:transform": [1, 0, 619037, 0, -1, 3430642, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0082", "private": {"created_by": "stac-task"}, "bbox": [-85.691535, 30.934103, -85.620917, 31.003395], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_ne_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008503_ne_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008503_ne_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.621902, 30.934103], [-85.620917, 31.002669], [-85.6906, 31.003395], [-85.691535, 30.934827], [-85.621902, 30.934103]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [625004, 3423113, 631658, 3430714], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7601, 6654], "eo:cloud_cover": 26, "proj:transform": [1, 0, 625004, 0, -1, 3430714, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0001", "private": {"created_by": "stac-task"}, "bbox": [-85.441706, 30.933975, -85.370747, 31.003522], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_ne_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008505_ne_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_ne_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.371913, 30.933975], [-85.370747, 31.00266], [-85.440589, 31.003522], [-85.441706, 30.934836], [-85.371913, 30.933975]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [648874, 3423421, 655544, 3431036], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7615, 6670], "eo:cloud_cover": 89, "proj:transform": [1, 0, 648874, 0, -1, 3431036, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0002", "private": {"created_by": "stac-task"}, "bbox": [-85.504167, 30.934008, -85.433293, 31.003486], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008505_nw_16_1_20110825.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008505_nw_16_1_20110825.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.434414, 30.934008], [-85.433293, 31.002658], [-85.503096, 31.003486], [-85.504167, 30.934834], [-85.434414, 30.934008]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-25T00:00:00Z", "naip:year": "2011", "proj:bbox": [642906, 3423339, 649572, 3430950], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7611, 6666], "eo:cloud_cover": 33, "proj:transform": [1, 0, 642906, 0, -1, 3430950, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0083", "private": {"created_by": "stac-task"}, "bbox": [-85.87891, 30.934198, -85.808547, 31.003302], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_nw_16_1_20110815.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008502_nw_16_1_20110815.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008502_nw_16_1_20110815.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.809397, 30.934198], [-85.808547, 31.002679], [-85.87811, 31.003302], [-85.87891, 30.934819], [-85.809397, 30.934198]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-15T00:00:00Z", "naip:year": "2011", "proj:bbox": [607102, 3422917, 613744, 3430508], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7591, 6642], "eo:cloud_cover": 26, "proj:transform": [1, 0, 607102, 0, -1, 3430508, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0086", "private": {"created_by": "stac-task"}, "bbox": [-87.816197, 30.87189, -87.746352, 30.940609], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_se_16_1_20110801.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_se_16_1_20110801.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_se_16_1_20110801.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746352, 30.872308], [-87.746882, 30.940609], [-87.816197, 30.94019], [-87.815618, 30.87189], [-87.746352, 30.872308]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2011", "proj:bbox": [422031, 3415689, 428653, 3423259], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7570, 6622], "eo:cloud_cover": 40, "proj:transform": [1, 0, 422031, 0, -1, 3423259, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0087", "private": {"created_by": "stac-task"}, "bbox": [-87.753661, 30.934419, -87.683886, 31.003078], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_nw_16_1_20110801.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_nw_16_1_20110801.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_nw_16_1_20110801.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683886, 30.934803], [-87.684373, 31.003078], [-87.753661, 31.002692], [-87.753125, 30.934419], [-87.683886, 30.934803]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2011", "proj:bbox": [428052, 3422577, 434667, 3430144], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7567, 6615], "eo:cloud_cover": 36, "proj:transform": [1, 0, 428052, 0, -1, 3430144, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0088", "private": {"created_by": "stac-task"}, "bbox": [-87.753652, 30.871926, -87.683891, 30.940576], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_sw_16_1_20110801.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008703_sw_16_1_20110801.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008703_sw_16_1_20110801.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683891, 30.87231], [-87.684377, 30.940576], [-87.753652, 30.94019], [-87.753117, 30.871926], [-87.683891, 30.87231]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2011", "proj:bbox": [428006, 3415651, 434624, 3423217], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6618], "eo:cloud_cover": 23, "proj:transform": [1, 0, 428006, 0, -1, 3423217, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0089", "private": {"created_by": "stac-task"}, "bbox": [-87.816182, 30.55939, -87.746368, 30.6281], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_ne_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_ne_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_ne_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746368, 30.559805], [-87.746892, 30.6281], [-87.816182, 30.627684], [-87.815609, 30.55939], [-87.746368, 30.559805]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421780, 3381056, 428421, 3388625], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6641], "eo:cloud_cover": 36, "proj:transform": [1, 0, 421780, 0, -1, 3388625, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0090", "private": {"created_by": "stac-task"}, "bbox": [-87.81619, 30.809396, -87.746349, 30.878106], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_ne_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_ne_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_ne_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746349, 30.809814], [-87.746878, 30.878106], [-87.81619, 30.877688], [-87.815612, 30.809396], [-87.746349, 30.809814]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421981, 3408763, 428607, 3416332], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6626], "eo:cloud_cover": 84, "proj:transform": [1, 0, 421981, 0, -1, 3416332, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0091", "private": {"created_by": "stac-task"}, "bbox": [-87.816185, 30.621895, -87.746357, 30.690605], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_se_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_se_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_se_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746357, 30.62231], [-87.746882, 30.690605], [-87.816185, 30.690188], [-87.815611, 30.621895], [-87.746357, 30.62231]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421830, 3387983, 428468, 3395552], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6638], "eo:cloud_cover": 31, "proj:transform": [1, 0, 421830, 0, -1, 3395552, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0092", "private": {"created_by": "stac-task"}, "bbox": [-87.816194, 30.746893, -87.746358, 30.815603], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_se_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008710_se_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008710_se_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746358, 30.74731], [-87.746885, 30.815603], [-87.816194, 30.815185], [-87.815618, 30.746893], [-87.746358, 30.74731]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421930, 3401836, 428560, 3409405], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7569, 6630], "eo:cloud_cover": 24, "proj:transform": [1, 0, 421930, 0, -1, 3409405, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0093", "private": {"created_by": "stac-task"}, "bbox": [-87.753639, 30.559423, -87.683911, 30.628074], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_nw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_nw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_nw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683911, 30.559805], [-87.68439, 30.628074], [-87.753639, 30.627692], [-87.753111, 30.559423], [-87.683911, 30.559805]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427774, 3381018, 434411, 3388584], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6637], "eo:cloud_cover": 61, "proj:transform": [1, 0, 427774, 0, -1, 3388584, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0094", "private": {"created_by": "stac-task"}, "bbox": [-87.753654, 30.809423, -87.683898, 30.878073], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_nw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_nw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_nw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683898, 30.809806], [-87.684382, 30.878073], [-87.753654, 30.877688], [-87.75312, 30.809423], [-87.683898, 30.809806]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427959, 3408724, 434581, 3416290], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6622], "eo:cloud_cover": 29, "proj:transform": [1, 0, 427959, 0, -1, 3416290, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0095", "private": {"created_by": "stac-task"}, "bbox": [-87.753646, 30.746928, -87.683895, 30.815579], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_sw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008711_sw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008711_sw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683895, 30.747312], [-87.684378, 30.815579], [-87.753646, 30.815194], [-87.753114, 30.746928], [-87.683895, 30.747312]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427913, 3401798, 434539, 3409364], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6626], "eo:cloud_cover": 63, "proj:transform": [1, 0, 427913, 0, -1, 3409364, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0096", "private": {"created_by": "stac-task"}, "bbox": [-87.753639, 30.684424, -87.683903, 30.753075], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_nw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_nw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_nw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683903, 30.684807], [-87.684385, 30.753075], [-87.753639, 30.752691], [-87.753109, 30.684424], [-87.683903, 30.684807]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427867, 3394871, 434496, 3402437], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7566, 6629], "eo:cloud_cover": 26, "proj:transform": [1, 0, 427867, 0, -1, 3402437, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0097", "private": {"created_by": "stac-task"}, "bbox": [-87.816189, 30.68439, -87.746357, 30.753109], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_ne_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008718_ne_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008718_ne_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746357, 30.684806], [-87.746883, 30.753109], [-87.816189, 30.752692], [-87.815614, 30.68439], [-87.746357, 30.684806]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [421880, 3394909, 428514, 3402479], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7570, 6634], "eo:cloud_cover": 1, "proj:transform": [1, 0, 421880, 0, -1, 3402479, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0098", "private": {"created_by": "stac-task"}, "bbox": [-87.753644, 30.621929, -87.683901, 30.69057], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_sw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008719_sw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008719_sw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683901, 30.622311], [-87.684382, 30.69057], [-87.753644, 30.690187], [-87.753115, 30.621929], [-87.683901, 30.622311]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427820, 3387945, 434454, 3395510], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7565, 6634], "eo:cloud_cover": 73, "proj:transform": [1, 0, 427820, 0, -1, 3395510, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0099", "private": {"created_by": "stac-task"}, "bbox": [-87.753635, 30.496927, -87.683911, 30.565569], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_sw_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008727_sw_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008727_sw_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.683911, 30.497308], [-87.684389, 30.565569], [-87.753635, 30.565186], [-87.753109, 30.496927], [-87.683911, 30.497308]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": "2011", "proj:bbox": [427728, 3374092, 434369, 3381657], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7565, 6641], "eo:cloud_cover": 90, "proj:transform": [1, 0, 427728, 0, -1, 3381657, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0084", "private": {"created_by": "stac-task"}, "bbox": [-85.316796, 30.93392, -85.245667, 31.003585], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_ne_16_1_20110802.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30085/m_3008506_ne_16_1_20110802.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30085/m_3008506_ne_16_1_20110802.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-85.246924, 30.93392], [-85.245667, 31.002654], [-85.315589, 31.003585], [-85.316796, 30.934848], [-85.246924, 30.93392]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-02T00:00:00Z", "naip:year": "2011", "proj:bbox": [660809, 3423596, 667487, 3431217], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "al", "proj:shape": [7621, 6678], "eo:cloud_cover": 52, "proj:transform": [1, 0, 660809, 0, -1, 3431217, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0085", "private": {"created_by": "stac-task"}, "bbox": [-87.816195, 30.934391, -87.746345, 31.00311], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_ne_16_1_20110801.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008702_ne_16_1_20110801.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008702_ne_16_1_20110801.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.746345, 30.93481], [-87.746877, 31.00311], [-87.816195, 31.002691], [-87.815614, 30.934391], [-87.746345, 30.93481]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-08-01T00:00:00Z", "naip:year": "2012", "proj:bbox": [422082, 3422616, 428700, 3430186], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "xx", "proj:shape": [7570, 6618], "eo:cloud_cover": 3, "proj:transform": [1, 0, 422082, 0, -1, 3430186, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} {"id": "pgstac-test-item-0100", "private": {"created_by": "stac-task"}, "bbox": [-87.816179, 30.496894, -87.74637, 30.565604], "type": "Feature", "links": [], "assets": {"image": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_se_16_1_20110731.tif", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "RGBIR COG tile", "eo:bands": [{"name": "Red", "common_name": "red"}, {"name": "Green", "common_name": "green"}, {"name": "Blue", "common_name": "blue"}, {"name": "NIR", "common_name": "nir", "description": "near-infrared"}]}, "metadata": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_fgdc_2011/30087/m_3008726_se_16_1_20110731.txt", "type": "text/plain", "roles": ["metadata"], "title": "FGDC Metdata"}, "thumbnail": {"href": "https://naipeuwest.blob.core.windows.net/naip/v002/al/2011/al_100cm_2011/30087/m_3008726_se_16_1_20110731.200.jpg", "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail"}}, "geometry": {"type": "Polygon", "coordinates": [[[-87.74637, 30.497308], [-87.746892, 30.565604], [-87.816179, 30.565188], [-87.815608, 30.496894], [-87.74637, 30.497308]]]}, "collection": "pgstac-test-collection", "properties": {"gsd": 1, "datetime": "2011-07-31T00:00:00Z", "naip:year": 2013, "proj:bbox": [421730, 3374130, 428375, 3381699], "proj:epsg": 26916, "providers": [{"url": "https://www.fsa.usda.gov/programs-and-services/aerial-photography/imagery-programs/naip-imagery/", "name": "USDA Farm Service Agency", "roles": ["producer", "licensor"]}], "naip:state": "zz", "proj:shape": [7569, 6645], "eo:cloud_cover": 50, "proj:transform": [1, 0, 421730, 0, -1, 3381699, 0, 0, 1]}, "stac_version": "1.0.0-beta.2", "stac_extensions": ["eo", "projection"]} ================================================ FILE: src/pypgstac/README.md ================================================ # pypgstac Python tools for working with PgSTAC ================================================ FILE: src/pypgstac/examples/load_queryables_example.py ================================================ #!/usr/bin/env python """ Example script demonstrating how to load queryables into PgSTAC. This script shows how to use the load_queryables function both from the command line and programmatically. """ import sys from pathlib import Path # Add the parent directory to the path so we can import pypgstac sys.path.append(str(Path(__file__).parent.parent)) from pypgstac.pypgstac import PgstacCLI def load_for_specific_collections( cli, sample_file, collection_ids, delete_missing=False, ): """Load queryables for specific collections. Args: cli: PgstacCLI instance sample_file: Path to the queryables file collection_ids: List of collection IDs to apply queryables to delete_missing: If True, delete properties not present in the file """ cli.load_queryables( str(sample_file), collection_ids=collection_ids, delete_missing=delete_missing, ) def main(): """Demonstrate loading queryables into PgSTAC.""" # Get the path to the sample queryables file sample_file = Path(__file__).parent / "sample_queryables.json" # Check if the file exists if not sample_file.exists(): return # Create a PgstacCLI instance # This will use the standard PostgreSQL environment variables for connection cli = PgstacCLI() # Load queryables for all collections cli.load_queryables(str(sample_file)) # Example of loading for specific collections load_for_specific_collections(cli, sample_file, ["landsat-8", "sentinel-2"]) # Example of loading queryables with delete_missing=True # This will delete properties not present in the file cli.load_queryables(str(sample_file), delete_missing=True) # Example of loading for specific collections with delete_missing=True # This will delete properties not present in the file, but only for the specified collections load_for_specific_collections( cli, sample_file, ["landsat-8", "sentinel-2"], delete_missing=True, ) if __name__ == "__main__": main() ================================================ FILE: src/pypgstac/examples/sample_queryables.json ================================================ { "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://example.com/stac/queryables", "type": "object", "title": "Queryables for Example STAC API", "description": "Queryable names for the Example STAC API", "properties": { "id": { "description": "Item identifier", "type": "string" }, "collection": { "description": "Collection identifier", "type": "string" }, "datetime": { "description": "Datetime", "type": "string", "format": "date-time" }, "geometry": { "description": "Geometry", "type": "object" }, "eo:cloud_cover": { "description": "Cloud cover percentage", "type": "number", "minimum": 0, "maximum": 100 }, "platform": { "description": "Platform name", "type": "string", "enum": ["landsat-8", "sentinel-2"] }, "instrument": { "description": "Instrument name", "type": "string" }, "gsd": { "description": "Ground sample distance in meters", "type": "number" }, "view:off_nadir": { "description": "Off-nadir angle in degrees", "type": "number" }, "view:sun_azimuth": { "description": "Sun azimuth angle in degrees", "type": "number" }, "view:sun_elevation": { "description": "Sun elevation angle in degrees", "type": "number" }, "sci:doi": { "description": "Digital Object Identifier", "type": "string" }, "created": { "description": "Date and time the item was created", "type": "string", "format": "date-time" }, "updated": { "description": "Date and time the item was last updated", "type": "string", "format": "date-time" }, "landcover:classes": { "description": "Land cover classes", "type": "array", "items": { "type": "string" } } }, "additionalProperties": true } ================================================ FILE: src/pypgstac/pyproject.toml ================================================ [project] name = "pypgstac" version = "0.9.11-dev" description = "Schema, functions and a python library for storing and accessing STAC collections and items in PostgreSQL" readme = "README.md" requires-python = ">=3.11" license = "MIT" authors = [{ name = "David Bitner", email = "bitner@dbspatial.com" }] keywords = ["STAC", "Postgresql", "PgSTAC"] classifiers = [ "Intended Audience :: Developers", "Intended Audience :: Information Technology", "Intended Audience :: Science/Research", "License :: OSI Approved :: MIT License", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", ] dependencies = [ "cachetools>=5.3.0", "fire>=0.7.0", "hydraters>=0.1.0", "orjson>=3.11.0", "plpygis>=0.5.0", "pydantic>=2.10,<3", "pydantic-settings>=2,<3", "python-dateutil>=2.8.0", "smart-open>=5.0", "tenacity>=8.1.0", "version-parser>=1.0.1", ] [project.optional-dependencies] test = [ "morecantile>=6.2,<7.1", "pytest>=8.3,<9.1", "pytest-benchmark>=5.1,<5.3", "pytest-cov>=6.0,<7.2", "pystac[validation]==1.*", "types-cachetools>=5.5", ] dev = [ "types-setuptools", "ruff==0.15.12", "ty==0.0.35", "pre-commit==4.6.0", ] psycopg = ["psycopg[binary]>=3.1.0", "psycopg-pool>=3.1.0"] migrations = ["psycopg2-binary", "migra"] docs = [ "jupyter", "pandas", "seaborn", "mkdocs-jupyter", "folium" ] [project.urls] Homepage = "https://stac-utils.github.io/pgstac/" Documentation = "https://stac-utils.github.io/pgstac/" Issues = "https://github.com/stac-utils/pgstac/issues" Source = "https://github.com/stac-utils/pgstac" Changelog = "https://stac-utils.github.io/pgstac/release-notes/" [project.scripts] pypgstac = "pypgstac.pypgstac:cli" [build-system] requires = ["hatchling"] build-backend = "hatchling.build" [tool.coverage.run] branch = true parallel = true [tool.coverage.report] exclude_lines = ["no cov", "if __name__ == .__main__.:", "if TYPE_CHECKING:"] [tool.ruff] target-version = "py311" line-length = 88 [tool.ruff.lint] select = [ "E", # pycodestyle errors "W", # pycodestyle warnings "F", # pyflakes "I", # isort "B", # flake8-bugbear "C4", # flake8-comprehensions "T20", # flake8-print "Q", # flake8-quotes "DTZ", # flake8-datetimez "ERA", # eradicate "PLC", "PLE", "PLW", "COM", # flake8-commas ] ignore = [ "E501", # line too long, handled by formatter "B008", # do not perform function calls in argument defaults "C901", # too complex "B905", ] [tool.ruff.lint.isort] known-first-party = ["pypgstac"] [tool.ty.environment] python-version = "3.11" [tool.ty.src] include = ["src/pypgstac", "tests"] [tool.pydocstyle] select = "D1" match = "(?!test).*.py" [tool.pytest.ini_options] addopts = "-vv --benchmark-skip" ================================================ FILE: src/pypgstac/src/pypgstac/__init__.py ================================================ """pyPgSTAC Version.""" from pypgstac.version import __version__ __all__ = ["__version__"] ================================================ FILE: src/pypgstac/src/pypgstac/db.py ================================================ """Base library for database interaction with PgSTAC.""" import atexit import logging import time from collections.abc import Generator from pathlib import Path from types import TracebackType from typing import Any import orjson import psycopg from psycopg import Connection, rows, sql from psycopg.abc import Params from psycopg.types import json as psycopg_json from psycopg.types.json import set_json_dumps, set_json_loads from psycopg_pool import ConnectionPool from pydantic_settings import BaseSettings, SettingsConfigDict from tenacity import retry, retry_if_exception_type, stop_after_attempt logger = logging.getLogger(__name__) def dumps(data: dict) -> str: """Dump dictionary as string.""" return orjson.dumps(data).decode() set_json_dumps(dumps) set_json_loads(orjson.loads) def pg_notice_handler(notice: psycopg.errors.Diagnostic) -> None: """Add PG messages to logging.""" msg = f"{notice.severity} - {notice.message_primary}" logger.info(msg) class Settings(BaseSettings): """Base Settings for Database Connection.""" db_min_conn_size: int = 0 db_max_conn_size: int = 1 db_max_queries: int = 5 db_max_idle: int = 5 db_num_workers: int = 1 db_retries: int = 3 model_config = SettingsConfigDict(env_file=Path(".env"), extra="ignore") settings = Settings() class PgstacDB: """Base class for interacting with PgSTAC Database.""" def __init__( self, dsn: str | None = "", pool: ConnectionPool | None = None, connection: Connection | None = None, commit_on_exit: bool = True, debug: bool = False, use_queue: bool = False, ) -> None: """Initialize Database.""" self.dsn: str if dsn is not None: self.dsn = dsn else: self.dsn = "" self.pool = pool self.connection = connection self.commit_on_exit = commit_on_exit self.initial_version = "0.1.9" self.debug = debug self.use_queue = use_queue if self.debug: logging.basicConfig(level=logging.DEBUG) def get_pool(self) -> ConnectionPool: """Get Database Pool.""" if self.pool is None: self.pool = ConnectionPool( conninfo=self.dsn, min_size=settings.db_min_conn_size, max_size=settings.db_max_conn_size, max_waiting=settings.db_max_queries, max_idle=settings.db_max_idle, num_workers=settings.db_num_workers, open=True, ) return self.pool def open(self) -> None: """Open database pool connection.""" self.get_pool() def close(self) -> None: """Close database pool connection.""" if self.pool is not None: self.pool.close() def connect(self) -> Connection: """Return database connection.""" pool = self.get_pool() if self.connection is None or self.connection.closed or self.connection.broken: self.connection = pool.getconn() self.connection.autocommit = True if self.debug: self.connection.add_notice_handler(pg_notice_handler) self.connection.execute( "SET CLIENT_MIN_MESSAGES TO NOTICE;", prepare=False, ) if self.use_queue: self.connection.execute( "SET pgstac.use_queue TO TRUE;", prepare=False, ) atexit.register(self.disconnect) self.connection.execute( """ SELECT CASE WHEN current_setting('search_path', false) ~* '\\mpgstac\\M' THEN current_setting('search_path', false) ELSE set_config( 'search_path', 'pgstac,' || current_setting('search_path', false), false ) END ; SET application_name TO 'pgstac'; """, prepare=False, ) return self.connection def wait(self) -> None: """Block until database connection is ready.""" cnt: int = 0 while cnt < 60: try: self.connect() self.query("SELECT 1;") return None except psycopg.errors.OperationalError: time.sleep(1) cnt += 1 raise psycopg.errors.CannotConnectNow def disconnect(self) -> None: """Disconnect from database.""" try: if self.connection is not None: if self.commit_on_exit: self.connection.commit() else: self.connection.rollback() except Exception: pass try: if self.pool is not None and self.connection is not None: self.pool.putconn(self.connection) except Exception: pass self.connection = None self.pool = None def __enter__(self) -> Any: """Enter used for context.""" self.connect() return self def __exit__( self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None, ) -> None: """Exit used for context.""" self.disconnect() @retry( stop=stop_after_attempt(settings.db_retries), retry=retry_if_exception_type(psycopg.errors.OperationalError), reraise=True, ) def query( self, query: Any, args: Params | None = None, row_factory: rows.BaseRowFactory = rows.tuple_row, ) -> Generator: """Query the database with parameters.""" conn = self.connect() try: with conn.cursor(row_factory=row_factory) as cursor: if args is None: rows = cursor.execute(query, prepare=False) else: rows = cursor.execute(query, args) if rows: for row in rows: yield row else: yield None except psycopg.errors.OperationalError as e: # If we get an operational error check the pool and retry logger.warning(f"OPERATIONAL ERROR: {e}") if self.pool is None: self.get_pool() else: self.pool.check() raise e except psycopg.errors.DatabaseError as e: if conn is not None: conn.rollback() raise e def query_one(self, *args: Any, **kwargs: Any) -> tuple[Any, ...] | str | None: """Return results from a query that returns a single row.""" try: r = next(self.query(*args, **kwargs)) except StopIteration: return None if r is None: return None if len(r) == 1: return r[0] return r def run_queued(self) -> str: try: self.connect().execute(""" CALL run_queued_queries(); """) return "Ran Queued Queries" except Exception as e: return f"Error Running Queued Queries: {e}" @property def version(self) -> str | None: """Get the current version number from a pgstac database.""" try: version = self.query_one( """ SELECT version from pgstac.migrations order by datetime desc, version desc limit 1; """, ) logger.debug(f"VERSION: {version}") if isinstance(version, bytes): version = version.decode() if isinstance(version, str): return version except psycopg.errors.UndefinedTable: logger.debug("PgSTAC is not installed.") if self.connection is not None: self.connection.rollback() return None @property def pg_version(self) -> str: """Get the current pg version number from a pgstac database.""" version = self.query_one( """ SHOW server_version_num; """, ) logger.debug(f"PG VERSION: {version}.") if isinstance(version, bytes): version = version.decode() if isinstance(version, str): if int(version) < 130000: major, minor, patch = tuple( map(int, [version[i : i + 2] for i in range(0, len(version), 2)]), ) raise Exception( f"PgSTAC requires PostgreSQL 13+, current version is: {major}.{minor}.{patch}", ) # noqa: E501 return version else: if self.connection is not None: self.connection.rollback() raise Exception("Could not find PG version.") def func(self, function_name: str, *args: Any) -> Generator: """Call a database function.""" placeholders = sql.SQL(", ").join(sql.Placeholder() * len(args)) func = sql.Identifier(function_name) cleaned_args = [] for arg in args: if isinstance(arg, dict): cleaned_args.append(psycopg_json.Jsonb(arg)) else: cleaned_args.append(arg) base_query = sql.SQL("SELECT * FROM {}({});").format(func, placeholders) return self.query(base_query, cleaned_args) def search(self, query: dict | str | psycopg_json.Jsonb = "{}") -> str: """Search PgSTAC.""" return dumps(next(self.func("search", query))[0]) ================================================ FILE: src/pypgstac/src/pypgstac/hydration.py ================================================ """Hydrate data in pypgstac rather than on the database.""" from copy import deepcopy from typing import Any, Mapping, cast from hydraters import hydrate # Marker value to indicate that a key should not be rehydrated DO_NOT_MERGE_MARKER = "𒍟※" def hydrate_py(base_item: dict[str, Any], item: dict[str, Any]) -> dict[str, Any]: """Hydrate item in-place with base_item properties. This will not perform a deep copy; values of the original item will be referenced in the return item. """ # Merge will mutate i, but create deep copies of values in the base item # This will prevent the base item values from being mutated, e.g. by # filtering out fields in `filter_fields`. def merge(b: dict[str, Any], i: dict[str, Any]) -> None: for key, _ in b.items(): if key in i: if isinstance(b[key], dict) and isinstance(i.get(key), dict): # Recurse on dicts to merge values merge(b[key], i[key]) elif isinstance(b[key], list) and isinstance(i.get(key), list): # Merge unequal lists, assume uniform types if len(b[key]) == len(i[key]): for bb, ii in zip(b[key], i[key]): # Make sure we're merging two dicts if isinstance(bb, dict) and isinstance(ii, dict): merge(bb, ii) else: # If item has a different length, then just use the item value continue else: # Key exists on item but isn't a dict or list, keep item value if i[key] == DO_NOT_MERGE_MARKER: # Key was marked as do-not-merge, drop it from the item del i[key] else: # Keep the item value continue else: # Keys in base item that are not in item are simply copied over i[key] = deepcopy(b[key]) merge(base_item, item) return item def dehydrate(base_item: dict[str, Any], full_item: dict[str, Any]) -> dict[str, Any]: """ Get a recursive difference between a base item and an incoming item to dehydrate. For keys of dicts within items, if the base item contains a key not present in the incoming item, then a do-no-merge value is added indicating that the key should not be rehydrated with the corresponding base item value. This will allow collection item-assets to contain keys that may not be present on individual items. """ def strip( base_value: Mapping[str, Any], item_value: Mapping[str, Any], ) -> dict[str, Any]: out: dict = {} for key, value in item_value.items(): if base_value is None or key not in base_value: # Nothing on base; preserve item value in the dehydrated item out[key] = value continue if base_value[key] == value: # Equal values; do not include in the dehydrated item continue if isinstance(base_value[key], list) and isinstance(value, list): if len(base_value[key]) == len(value): # Equal length lists dehydrate dicts at each matching index # and use incoming item values for other types out[key] = [] for bv, v in zip(base_value[key], value): if isinstance(bv, dict) and isinstance(v, dict): bv_mapping = cast(Mapping[str, Any], bv) v_mapping = cast(Mapping[str, Any], v) dehydrated = strip(bv_mapping, v_mapping) apply_marked_keys(bv_mapping, v_mapping, dehydrated) out[key].append(dehydrated) else: out[key].append(v) else: # Unequal length lists are not dehydrated and just use the # incoming item value out[key] = value continue if value is None or value == []: # Don't keep empty values continue if isinstance(value, dict) and isinstance(base_value[key], Mapping): # After dehdrating a dict, mark any keys that are present on the # base item but not in the incoming item as `do-not-merge` during # rehydration dehydrated = strip(base_value[key], value) apply_marked_keys(base_value[key], value, dehydrated) out[key] = dehydrated continue else: # Unequal non-dict values are copied over from the incoming item out[key] = value # Mark any top-level keys from the base_item that are not in the incoming item apply_marked_keys(base_value, item_value, out) return out return strip(base_item, full_item) def apply_marked_keys( base_item: Mapping[str, Any], full_item: Mapping[str, Any], dehydrated: dict[str, Any], ) -> None: """Mark keys. Mark any keys that are present on the base item but not in the incoming item as `do-not-merge` on the dehydrated item. This will prevent they key from being rehydrated. This modifies the dehydrated item in-place. """ try: marked_keys = [key for key in base_item if key not in full_item] marked_dict = dict.fromkeys(marked_keys, DO_NOT_MERGE_MARKER) dehydrated.update(marked_dict) except TypeError: pass __all__ = [ "apply_marked_keys", "dehydrate", "hydrate", "hydrate_py", ] ================================================ FILE: src/pypgstac/src/pypgstac/load.py ================================================ """Utilities to bulk load data into pgstac from json/ndjson.""" import contextlib import itertools import logging import re import sys import time from dataclasses import dataclass from datetime import UTC, datetime from enum import Enum from pathlib import Path from typing import ( Any, BinaryIO, Generator, Iterable, Iterator, TextIO, ) import orjson import psycopg from cachetools.func import lru_cache from orjson import JSONDecodeError from plpygis.geometry import Geometry from psycopg import sql from smart_open import open from tenacity import ( retry, retry_if_exception_type, stop_after_attempt, wait_random_exponential, ) from version_parser import Version as V from .db import PgstacDB from .hydration import dehydrate from .version import __version__ logger = logging.getLogger(__name__) MIN_DATETIME_UTC = datetime.min.replace(tzinfo=UTC) MAX_DATETIME_UTC = datetime.max.replace(tzinfo=UTC) def _normalize_version_for_parse(version: str) -> str: """Extract the numeric semver prefix for version_parser compatibility.""" match = re.match(r"^(\d+\.\d+\.\d+)", str(version)) if match is not None: return match.group(1) return str(version) @dataclass class Partition: name: str collection: str datetime_range_min: str datetime_range_max: str end_datetime_range_min: str end_datetime_range_max: str requires_update: bool def chunked_iterable(iterable: Iterable, size: int | None = 10000) -> Iterable: """Chunk an iterable.""" it = iter(iterable) while True: chunk = tuple(itertools.islice(it, size)) if not chunk: break yield chunk class Tables(str, Enum): """Available tables for loading.""" items = "items" collections = "collections" class Methods(str, Enum): """Available methods for loading data.""" insert = "insert" ignore = "ignore" upsert = "upsert" delsert = "delsert" insert_ignore = "insert_ignore" @contextlib.contextmanager def open_std( filename: str, mode: str = "r", *args: Any, **kwargs: Any, ) -> Generator[Any, None, None]: """Open files and i/o streams transparently.""" fh: TextIO | BinaryIO if ( filename is None or filename == "-" or filename == "stdin" or filename == "stdout" ): stream = sys.stdin if "r" in mode else sys.stdout fh = stream.buffer if "b" in mode else stream close = False else: fh = open(filename, mode, *args, **kwargs) close = True try: yield fh finally: if close: try: fh.close() except AttributeError: pass def read_json(file: Path | str | Iterator[Any] = "stdin") -> Iterable: """Load data from an ndjson or json file.""" if file is None: file = "stdin" if isinstance(file, str): open_file: Any = open_std(file, "r") with open_file as f: # Try reading line by line as ndjson try: for line in f: lineout = line.strip().replace("\\\\", "\\").replace("\\\\", "\\") yield orjson.loads(lineout) except JSONDecodeError: # If reading first line as json fails, try reading entire file logger.info("First line could not be parsed as json, trying full file.") try: f.seek(0) json = orjson.loads(f.read()) if isinstance(json, list): for record in json: yield record else: yield json except JSONDecodeError: logger.info("File cannot be read as json") raise elif isinstance(file, Iterable): for line in file: if isinstance(line, dict): yield line elif isinstance(line, (str, bytes, bytearray, memoryview)): yield orjson.loads(line) else: raise TypeError("Unsupported json input type in iterable.") class Loader: """Utilities for loading data.""" db: PgstacDB _partition_cache: dict[str, Partition] def __init__(self, db: PgstacDB): self.db = db self._partition_cache: dict[str, Partition] = {} def check_version(self) -> None: db_version = self.db.version if db_version is None: raise Exception("Failed to detect the target database version.") if db_version != "unreleased": v1 = V(_normalize_version_for_parse(db_version)) v2 = V(_normalize_version_for_parse(__version__)) if (v1.get_major_version(), v1.get_minor_version()) != ( v2.get_major_version(), v2.get_minor_version(), ): raise Exception( f"pypgstac version {__version__}" " is not compatible with the target" f" database version {self.db.version}." f" database version {db_version}.", ) @lru_cache(maxsize=128) def collection_json(self, collection_id: str) -> tuple[dict[str, Any], int, str]: """Get collection.""" res = self.db.query_one( "SELECT base_item, key, partition_trunc FROM collections WHERE id=%s", (collection_id,), ) if isinstance(res, tuple): base_item, key, partition_trunc = res else: raise Exception(f"Error getting info for {collection_id}.") if key is None: raise Exception( f"Collection {collection_id} is not present in the database", ) logger.debug(f"Found {collection_id} with base_item {base_item}") return base_item, key, partition_trunc def load_collections( self, file: Path | str | Iterator[Any] = "stdin", insert_mode: Methods | None = Methods.insert, ) -> None: """Load a collections json or ndjson file.""" self.check_version() if file is None: file = "stdin" conn = self.db.connect() with conn.cursor() as cur: with conn.transaction(): cur.execute( """ DROP TABLE IF EXISTS tmp_collections; CREATE TEMP TABLE tmp_collections (content jsonb) ON COMMIT DROP; """, ) with cur.copy("COPY tmp_collections (content) FROM stdin;") as copy: for collection in read_json(file): copy.write_row((orjson.dumps(collection).decode(),)) if insert_mode in ( None, Methods.insert, ): cur.execute( """ INSERT INTO collections (content) SELECT content FROM tmp_collections; """, ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") elif insert_mode in ( Methods.insert_ignore, Methods.ignore, ): cur.execute( """ INSERT INTO collections (content) SELECT content FROM tmp_collections ON CONFLICT DO NOTHING; """, ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") elif insert_mode == Methods.upsert: cur.execute( """ INSERT INTO collections (content) SELECT content FROM tmp_collections ON CONFLICT (id) DO UPDATE SET content=EXCLUDED.content; """, ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") else: raise Exception( "Available modes are insert, ignore, and upsert." f"You entered {insert_mode}.", ) @retry( stop=stop_after_attempt(10), wait=wait_random_exponential(multiplier=1, max=120), retry=( retry_if_exception_type(psycopg.errors.CheckViolation) | retry_if_exception_type(psycopg.errors.DeadlockDetected) | retry_if_exception_type(psycopg.errors.SerializationFailure) | retry_if_exception_type(psycopg.errors.LockNotAvailable) | retry_if_exception_type(psycopg.errors.ObjectInUse) ), reraise=True, before_sleep=lambda retry_state: ( setattr( retry_state.args[1], "requires_update", True, ) if retry_state.outcome is not None and isinstance( retry_state.outcome.exception(), psycopg.errors.CheckViolation, ) else None ), ) def load_partition( self, partition: Partition, items: Iterable[dict[str, Any]], insert_mode: Methods | None = Methods.insert, ) -> None: """Load items data for a single partition.""" conn = self.db.connect() t = time.perf_counter() logger.debug(f"Loading data for partition: {partition}.") with conn.cursor() as cur: if partition.requires_update: with conn.transaction(): cur.execute( """ SELECT check_partition( %s, tstzrange(%s, %s, '[]'), tstzrange(%s, %s, '[]') ); """, ( partition.collection, partition.datetime_range_min, partition.datetime_range_max, partition.end_datetime_range_min, partition.end_datetime_range_max, ), ) logger.debug( f"Adding or updating partition {partition.name} " f"took {time.perf_counter() - t}s", ) partition.requires_update = False else: logger.debug( f"Partition {partition.name} does not require an update.", ) with conn.transaction(): t = time.perf_counter() if insert_mode in ( None, Methods.insert, ): with cur.copy( sql.SQL( """ COPY {} (id, collection, datetime, end_datetime, geometry, content, private) FROM stdin; """, ).format(sql.Identifier(partition.name)), ) as copy: for item in items: item.pop("partition", None) copy.write_row( ( item["id"], item["collection"], item["datetime"], item["end_datetime"], item["geometry"], item["content"], item.get("private", None), ), ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") elif insert_mode in ( Methods.insert_ignore, Methods.upsert, Methods.delsert, Methods.ignore, ): cur.execute( """ DROP TABLE IF EXISTS items_ingest_temp; CREATE TEMP TABLE items_ingest_temp ON COMMIT DROP AS SELECT * FROM items LIMIT 0; """, ) with cur.copy( """ COPY items_ingest_temp (id, collection, datetime, end_datetime, geometry, content, private) FROM stdin; """, ) as copy: for item in items: item.pop("partition", None) copy.write_row( ( item["id"], item["collection"], item["datetime"], item["end_datetime"], item["geometry"], item["content"], item.get("private", None), ), ) logger.debug(cur.statusmessage) logger.debug(f"Copied rows: {cur.rowcount}") cur.execute( sql.SQL( """ LOCK TABLE ONLY {} IN EXCLUSIVE MODE; """, ).format(sql.Identifier(partition.name)), ) if insert_mode in ( Methods.ignore, Methods.insert_ignore, ): cur.execute( sql.SQL( """ INSERT INTO {} SELECT * FROM items_ingest_temp ON CONFLICT DO NOTHING; """, ).format(sql.Identifier(partition.name)), ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") elif insert_mode == Methods.upsert: cur.execute( sql.SQL( """ INSERT INTO {} AS t SELECT * FROM items_ingest_temp ON CONFLICT (id) DO UPDATE SET datetime = EXCLUDED.datetime, end_datetime = EXCLUDED.end_datetime, geometry = EXCLUDED.geometry, collection = EXCLUDED.collection, content = EXCLUDED.content WHERE t IS DISTINCT FROM EXCLUDED ; """, ).format(sql.Identifier(partition.name)), ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") elif insert_mode == Methods.delsert: cur.execute( sql.SQL( """ WITH deletes AS ( DELETE FROM items i USING items_ingest_temp s WHERE i.id = s.id AND i.collection = s.collection ) INSERT INTO {} AS t SELECT * FROM items_ingest_temp ON CONFLICT (id) DO UPDATE SET datetime = EXCLUDED.datetime, end_datetime = EXCLUDED.end_datetime, geometry = EXCLUDED.geometry, collection = EXCLUDED.collection, content = EXCLUDED.content WHERE t IS DISTINCT FROM EXCLUDED ; """, ).format(sql.Identifier(partition.name)), ) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") else: raise Exception( "Available modes are insert, ignore, upsert, and delsert." f"You entered {insert_mode}.", ) logger.debug("Updating Partition Stats") cur.execute("SELECT update_partition_stats_q(%s);", (partition.name,)) logger.debug(cur.statusmessage) logger.debug(f"Rows affected: {cur.rowcount}") logger.debug( f"Copying data for {partition} took {time.perf_counter() - t} seconds", ) def _partition_update(self, item: dict[str, Any]) -> str: """Update the cached partition with the item information and return the name. This method will mark the partition as dirty if the bounds of the partition need to be updated based on this item. """ p = item.get("partition", None) if p is None: _, key, partition_trunc = self.collection_json(item["collection"]) if partition_trunc == "year": pd = item["datetime"].replace("-", "")[:4] p = f"_items_{key}_{pd}" elif partition_trunc == "month": pd = item["datetime"].replace("-", "")[:6] p = f"_items_{key}_{pd}" else: p = f"_items_{key}" item["partition"] = p partition_name: str = p partition: Partition | None = None if partition_name not in self._partition_cache: # Read the partition information from the database if it exists db_rows = list( self.db.query( """ SELECT nullif(lower(constraint_dtrange),'-infinity') as datetime_range_min, nullif(upper(constraint_dtrange),'infinity') as datetime_range_max, nullif(lower(constraint_edtrange),'-infinity') as end_datetime_range_min, nullif(upper(constraint_edtrange),'infinity') as end_datetime_range_max FROM partition_sys_meta WHERE partition=%s; """, [partition_name], ), ) if db_rows: datetime_range_min: datetime = db_rows[0][0] or MIN_DATETIME_UTC datetime_range_max: datetime = db_rows[0][1] or MAX_DATETIME_UTC end_datetime_range_min: datetime = db_rows[0][2] or MIN_DATETIME_UTC end_datetime_range_max: datetime = db_rows[0][3] or MAX_DATETIME_UTC partition = Partition( name=partition_name, collection=item["collection"], datetime_range_min=datetime_range_min.isoformat(), datetime_range_max=datetime_range_max.isoformat(), end_datetime_range_min=end_datetime_range_min.isoformat(), end_datetime_range_max=end_datetime_range_max.isoformat(), requires_update=False, ) else: partition = self._partition_cache[partition_name] if partition: # Only update the partition if the item is outside the current bounds if item["datetime"] < partition.datetime_range_min: partition.datetime_range_min = item["datetime"] partition.requires_update = True if item["datetime"] > partition.datetime_range_max: partition.datetime_range_max = item["datetime"] partition.requires_update = True if item["end_datetime"] < partition.end_datetime_range_min: partition.end_datetime_range_min = item["end_datetime"] partition.requires_update = True if item["end_datetime"] > partition.end_datetime_range_max: partition.end_datetime_range_max = item["end_datetime"] partition.requires_update = True else: # No partition exists yet; create a new one from item partition = Partition( name=partition_name, collection=item["collection"], datetime_range_min=item["datetime"], datetime_range_max=item["datetime"], end_datetime_range_min=item["end_datetime"], end_datetime_range_max=item["end_datetime"], requires_update=True, ) self._partition_cache[partition_name] = partition return partition_name def read_dehydrated(self, file: Path | str = "stdin") -> Generator: if file is None: file = "stdin" if isinstance(file, str): open_file: Any = open_std(file, "r") with open_file as f: # Note: if 'content' is changed to be anything # but the last field, the logic below will break. fields = [ "id", "geometry", "collection", "datetime", "end_datetime", "content", ] for line in f: tab_split = line.split("\t") item = {} for i, field in enumerate(fields): if field == "content": # Join the remaining splits in case # there were any tabs in the JSON content. content_value = "\t".join(tab_split[i:]) # Replace quote characters that can be # written on export and causes failures. content_value = content_value.replace(r'\\"', r"\"") item[field] = content_value else: item[field] = tab_split[i] item["partition"] = self._partition_update(item) yield item def read_hydrated( self, file: Path | str | Iterator[Any] = "stdin", ) -> Generator: for line in read_json(file): item = self.format_item(line) item["partition"] = self._partition_update(item) yield item def load_items( self, file: Path | str | Iterator[Any] = "stdin", insert_mode: Methods | None = Methods.insert, dehydrated: bool | None = False, chunksize: int | None = 10000, ) -> None: """Load items json records.""" self.check_version() if file is None: file = "stdin" t = time.perf_counter() self._partition_cache = {} if dehydrated and isinstance(file, str): items = self.read_dehydrated(file) else: items = self.read_hydrated(file) for chunkin in chunked_iterable(items, chunksize): chunk = list(chunkin) chunk.sort(key=lambda x: x["partition"]) for k, g in itertools.groupby(chunk, lambda x: x["partition"]): self.load_partition(self._partition_cache[k], list(g), insert_mode) logger.debug(f"Adding data to database took {time.perf_counter() - t} seconds.") def format_item(self, _item: Path | str | dict[str, Any]) -> dict[str, Any]: """Format an item to insert into a record.""" out: dict[str, Any] = {} item: dict[str, Any] if not isinstance(_item, dict): try: item = orjson.loads(str(_item).replace("\\\\", "\\")) except Exception: raise else: item = _item base_item, key, partition_trunc = self.collection_json(item["collection"]) out["id"] = item.get("id") out["collection"] = item.get("collection") properties: dict[str, Any] = item.get("properties", {}) dt: str | None = properties.get("datetime") edt: str | None = properties.get("end_datetime") sdt: str | None = properties.get("start_datetime") if edt is not None and sdt is not None: out["datetime"] = sdt out["end_datetime"] = edt elif dt is not None: out["datetime"] = dt out["end_datetime"] = dt else: raise Exception("Invalid datetime encountered") if out["datetime"] is None or out["end_datetime"] is None: raise Exception( f"Datetime must be set. OUT: {out} Properties: {properties}", ) if partition_trunc == "year": pd = out["datetime"].replace("-", "")[:4] partition = f"_items_{key}_{pd}" elif partition_trunc == "month": pd = out["datetime"].replace("-", "")[:6] partition = f"_items_{key}_{pd}" else: partition = f"_items_{key}" out["partition"] = partition geojson = item.get("geometry") if geojson is None: geometry = None else: geom = Geometry.from_geojson(geojson) if geom is None: raise Exception(f"Invalid geometry encountered: {geojson}") geometry = str(geom.ewkb) out["geometry"] = geometry content = dehydrate(base_item, item) # Remove keys from the dehydrated item content which are stored directly # on the table row. content.pop("id", None) content.pop("collection", None) content.pop("geometry", None) if (private := content.pop("private", None)) is not None: out["private"] = orjson.dumps(private).decode() else: out["private"] = None out["content"] = orjson.dumps(content).decode() return out def __hash__(self) -> int: """Return hash so that the LRU deocrator can cache without the class.""" return 0 ================================================ FILE: src/pypgstac/src/pypgstac/migrate.py ================================================ """Utilities to help migrate pgstac schema.""" import glob import logging import os import re from collections import defaultdict from collections.abc import Iterator from typing import Any, cast from smart_open import open from . import __version__ from .db import PgstacDB dirname = os.path.dirname(__file__) migrations_dir = os.path.join(dirname, "migrations") logger = logging.getLogger(__name__) class MigrationPath: """Calculate path from migration files to get from one version to the next.""" def __init__(self, path: str, f: str, t: str) -> None: """Initialize MigrationPath.""" self.path = path if f is None: f = "init" if t is None: raise Exception('Must set "to" version') if f == t: raise Exception("No Migration Necessary") self.f = f self.t = t def parse_filename(self, filename: str) -> list[str]: """Get version numbers from filename.""" filename = os.path.splitext(os.path.basename(filename))[0].replace( "pgstac.", "", ) return filename.split("-") def get_files(self) -> Iterator[str]: """Find all migration files available.""" path = self.path.rstrip("/") return glob.iglob(f"{path}/*.sql") def build_graph(self) -> dict[str, list[str]]: """Build a graph to get from one version to another.""" graph = defaultdict(list) for file in self.get_files(): parts = self.parse_filename(file) if len(parts) == 2: graph[parts[0]].append(parts[1]) else: graph["init"].append(parts[0]) return graph def build_path(self) -> list[str] | None: """Create the path of ordered files needed to migrate.""" graph = self.build_graph() explored: list[str] = [] q = [[self.f]] while q: path = q.pop(0) node = path[-1] if node not in explored: neighbours = graph[node] for neighbour in neighbours: new_path = list(path) new_path.append(neighbour) q.append(new_path) if neighbour == self.t: return new_path explored.append(node) return None def migrations(self) -> list[str]: """Return the list of migrations needed in order.""" path = self.build_path() if path is None: raise Exception( f"Could not determine path to get from {self.f} to {self.t}.", ) if len(path) == 1: return [f"pgstac.{path[0]}.sql"] files = [] for idx in range(len(path) - 1): f = f"pgstac.{path[idx]}-{path[idx + 1]}.sql" f = f.replace("--init", "") files.append(f"pgstac.{path[idx]}-{path[idx + 1]}.sql") return files def get_sql(file: str) -> str: """Get sql from a file as a string.""" sqlstrs = [] file = re.sub("[0-9]+[.][0-9]+[.][0-9]+-dev", "unreleased", file) fp = os.path.join(migrations_dir, file) file_handle: Any = open(fp) with file_handle as fd: sqlstrs.extend(fd.readlines()) return "\n".join(sqlstrs) class Migrate: """Utilities for migrating pgstac database.""" def __init__(self, db: PgstacDB, schema: str = "pgstac"): """Prepare for migration.""" self.db = db self.schema = schema def run_migration(self, toversion: str | None = None) -> str: """Migrate a pgstac database to current version.""" if toversion is None: toversion = __version__ files = [] if re.search(r"-dev$", toversion): logger.info("using unreleased version") toversion = "unreleased" major, minor, patch = tuple( map( int, [ self.db.pg_version[i : i + 2] for i in range(0, len(self.db.pg_version), 2) ], ), ) logger.info(f"Migrating PgSTAC on PostgreSQL Version {major}.{minor}.{patch}") oldversion = self.db.version if oldversion == toversion: logger.info(f"Target database already at version: {toversion}") return toversion if oldversion is None: logger.info(f"No pgstac version set, installing {toversion} from scratch.") files.append(os.path.join(migrations_dir, f"pgstac.{toversion}.sql")) else: logger.info(f"Migrating from {oldversion} to {toversion}.") m = MigrationPath(migrations_dir, oldversion, toversion) files = m.migrations() if len(files) < 1: raise Exception("Could not find migration files") conn = self.db.connect() with conn.cursor() as cur: conn.autocommit = False for file in files: logger.debug(f"Running migration file {file}.") migration_sql = get_sql(file) # Migration SQL is loaded from trusted local migration files. cur.execute(cast(Any, migration_sql)) logger.debug(cur.statusmessage) logger.debug(cur.rowcount) logger.debug(f"Database migrated to {toversion}") newversion = self.db.version if conn is not None: if newversion == toversion: conn.commit() else: conn.rollback() raise Exception( "Migration failed, database rolled back to previous state.", ) logger.debug(f"New Version: {newversion}") if newversion is None: raise Exception("Migration failed to report a new version.") return newversion ================================================ FILE: src/pypgstac/src/pypgstac/py.typed ================================================ ================================================ FILE: src/pypgstac/src/pypgstac/pypgstac.py ================================================ """Command utilities for managing pgstac.""" import logging import sys import fire import orjson from smart_open import open from pypgstac.db import PgstacDB from pypgstac.load import Loader, Methods, Tables, read_json from pypgstac.migrate import Migrate class PgstacCLI: """CLI for PgSTAC.""" def __init__( self, dsn: str | None = "", version: bool = False, debug: bool = False, usequeue: bool = False, ): """Initialize PgSTAC CLI.""" if version: sys.exit(0) self.dsn = dsn self._db = PgstacDB(dsn=dsn, debug=debug, use_queue=usequeue) if debug: logging.basicConfig(level=logging.DEBUG) sys.tracebacklimit = 1000 @property def initversion(self) -> str: """Return earliest migration version.""" return "0.1.9" @property def version(self) -> str | None: """Get PgSTAC version installed on database.""" return self._db.version @property def pg_version(self) -> str: """Get PostgreSQL server version installed on database.""" return self._db.pg_version def pgready(self) -> None: """Wait for a pgstac database to accept connections.""" self._db.wait() def search(self, query: str) -> str: """Search PgSTAC.""" return self._db.search(query) def migrate(self, toversion: str | None = None) -> str: """Migrate PgSTAC Database.""" migrator = Migrate(self._db) return migrator.run_migration(toversion=toversion) def load( self, table: Tables, file: str, method: Methods | None = Methods.insert, dehydrated: bool | None = False, chunksize: int | None = 10000, ) -> None: """Load collections or items into PgSTAC.""" loader = Loader(db=self._db) if table == "collections": loader.load_collections(file, method) if table == "items": loader.load_items(file, method, dehydrated, chunksize) def runqueue(self) -> str: return self._db.run_queued() def loadextensions(self) -> None: conn = self._db.connect() with conn.cursor() as cur: cur.execute( """ INSERT INTO stac_extensions (url) SELECT DISTINCT substring( jsonb_array_elements_text(content->'stac_extensions') FROM E'^[^#]*' ) FROM collections ON CONFLICT DO NOTHING; """, ) conn.commit() urls = self._db.query( """ SELECT url FROM stac_extensions WHERE content IS NULL; """, ) if urls: for u in urls: url = u[0] try: with open(url, "r") as f: content = f.read() self._db.query( """ UPDATE pgstac.stac_extensions SET content=%s WHERE url=%s ; """, [content, url], ) conn.commit() except Exception: pass def load_queryables( self, file: str, collection_ids: list[str] | None = None, delete_missing: bool | None = False, index_fields: list[str] | None = None, ) -> None: """Load queryables from a JSON file. Args: file: Path to the JSON file containing queryables definition collection_ids: Comma-separated list of collection IDs to apply the queryables to delete_missing: If True, delete properties not present in the file. If collection_ids is specified, only delete properties for those collections. index_fields: List of field names to create indexes for. If not provided, no indexes will be created. Creating too many indexes can negatively impact performance. """ # Read the queryables JSON file queryables_data = None for item in read_json(file): queryables_data = item break # We only need the first item if not queryables_data: raise ValueError(f"No valid JSON data found in {file}") # Extract properties from the queryables definition properties = queryables_data.get("properties", {}) if not properties: raise ValueError("No properties found in queryables definition") conn = self._db.connect() with conn.cursor() as cur: with conn.transaction(): # Insert each property as a queryable for name, definition in properties.items(): # Skip core fields that are already indexed if name in ( "id", "geometry", "datetime", "end_datetime", "collection", ): continue # Determine property wrapper based on type property_wrapper = "to_text" # default if definition.get("type") == "number": property_wrapper = "to_float" elif definition.get("type") == "integer": property_wrapper = "to_int" elif definition.get("format") == "date-time": property_wrapper = "to_tstz" elif definition.get("type") == "array": property_wrapper = "to_text_array" # Determine if this field should be indexed property_index_type = None if index_fields and name in index_fields: property_index_type = "BTREE" # First delete any existing queryable with the same name if not collection_ids: # If no collection_ids specified, delete queryables # with NULL collection_ids cur.execute( """ DELETE FROM queryables WHERE name = %s AND collection_ids IS NULL """, [name], ) else: # Delete queryables with matching name and collection_ids cur.execute( """ DELETE FROM queryables WHERE name = %s AND collection_ids = %s::text[] """, [name, collection_ids], ) # Also delete queryables with NULL collection_ids cur.execute( """ DELETE FROM queryables WHERE name = %s AND collection_ids IS NULL """, [name], ) # Then insert the new queryable cur.execute( """ INSERT INTO queryables (name, collection_ids, definition, property_wrapper, property_index_type) VALUES (%s, %s, %s, %s, %s) """, [ name, collection_ids, orjson.dumps(definition).decode(), property_wrapper, property_index_type, ], ) # If delete_missing is True, # delete all queryables that were not in the file if delete_missing: # Get the list of property names from the file property_names = list(properties.keys()) # Skip core fields that are already indexed core_fields = [ "id", "geometry", "datetime", "end_datetime", "collection", ] property_names = [ name for name in property_names if name not in core_fields ] if not property_names: # If no valid properties, don't delete anything pass elif not collection_ids: # If no collection_ids specified, # delete queryables with NULL collection_ids # that are not in the property_names list placeholders = ", ".join(["%s"] * len(property_names)) core_placeholders = ", ".join(["%s"] * len(core_fields)) # Build the query with proper placeholders query = f""" DELETE FROM queryables WHERE collection_ids IS NULL AND name NOT IN ({placeholders}) AND name NOT IN ({core_placeholders}) """ # Flatten the parameters params = property_names + core_fields cur.execute(query, params) else: # Delete queryables with matching collection_ids # that are not in the property_names list placeholders = ", ".join(["%s"] * len(property_names)) core_placeholders = ", ".join(["%s"] * len(core_fields)) # Build the query with proper placeholders query = f""" DELETE FROM queryables WHERE collection_ids = %s::text[] AND name NOT IN ({placeholders}) AND name NOT IN ({core_placeholders}) """ # Flatten the parameters params = [collection_ids] + property_names + core_fields cur.execute(query, params) # Trigger index creation only if index_fields were provided if index_fields and len(index_fields) > 0: cur.execute("SELECT maintain_partitions();") def cli() -> None: """Wrap fire call for CLI.""" fire.Fire(PgstacCLI) if __name__ == "__main__": fire.Fire(PgstacCLI) ================================================ FILE: src/pypgstac/src/pypgstac/version.py ================================================ """Version.""" __version__ = "0.9.11-dev" ================================================ FILE: src/pypgstac/tests/__init__.py ================================================ ================================================ FILE: src/pypgstac/tests/conftest.py ================================================ """Fixtures for pypgstac tests.""" import os from typing import Generator import psycopg import pytest from pypgstac.db import PgstacDB from pypgstac.load import Loader from pypgstac.migrate import Migrate @pytest.fixture(scope="function") def db() -> Generator: """Fixture to get a fresh database.""" origdb: str = os.getenv("PGDATABASE", "") with psycopg.connect(autocommit=True) as conn: try: conn.execute( """ CREATE DATABASE pypgstactestdb TEMPLATE pgstac_test_db_template; """, ) except psycopg.errors.DuplicateDatabase: try: conn.execute( """ DROP DATABASE pypgstactestdb WITH (FORCE); """, ) conn.execute( """ CREATE DATABASE pypgstactestdb TEMPLATE pgstac_test_db_template; """, ) except psycopg.errors.InsufficientPrivilege: try: conn.execute("DROP DATABASE pypgstactestdb;") conn.execute( """ CREATE DATABASE pypgstactestdb TEMPLATE pgstac_test_db_template; """, ) except Exception: pass os.environ["PGDATABASE"] = "pypgstactestdb" pgdb = PgstacDB() yield pgdb pgdb.close() os.environ["PGDATABASE"] = origdb with psycopg.connect(autocommit=True) as conn: try: conn.execute("DROP DATABASE pypgstactestdb WITH (FORCE);") except psycopg.errors.InsufficientPrivilege: try: conn.execute("DROP DATABASE pypgstactestdb;") except Exception: pass @pytest.fixture(scope="function") def loader(db: PgstacDB) -> Loader: """Fixture to get a loader and an empty pgstac.""" if False: db.query("DROP SCHEMA IF EXISTS pgstac CASCADE;") Migrate(db).run_migration() ldr = Loader(db) return ldr ================================================ FILE: src/pypgstac/tests/data-files/hydration/collections/chloris-biomass.json ================================================ { "id": "chloris-biomass", "type": "Collection", "links": [ { "rel": "items", "type": "application/geo+json", "href": "https://planetarycomputer.microsoft.com/api/stac/v1/collections/chloris-biomass/items" }, { "rel": "parent", "type": "application/json", "href": "https://planetarycomputer.microsoft.com/api/stac/v1/" }, { "rel": "root", "type": "application/json", "href": "https://planetarycomputer.microsoft.com/api/stac/v1/" }, { "rel": "self", "type": "application/json", "href": "https://planetarycomputer.microsoft.com/api/stac/v1/collections/chloris-biomass" }, { "rel": "license", "href": "https://spdx.org/licenses/CC-BY-NC-SA-4.0.html", "title": "Creative Commons Attribution Non Commercial Share Alike 4.0 International" }, { "rel": "describedby", "href": "https://planetarycomputer.microsoft.com/dataset/chloris-biomass", "title": "Human readable dataset overview and reference", "type": "text/html" } ], "title": "Chloris Biomass", "assets": { "thumbnail": { "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/chloris-biomass.jpg", "type": "image/jpg", "roles": [ "thumbnail" ], "title": "Chloris Biomass" } }, "extent": { "spatial": { "bbox": [ [ -179.95, -60, 179.95, 90 ] ] }, "temporal": { "interval": [ [ "2003-07-31T00:00:00Z", "2019-07-31T00:00:00Z" ] ] } }, "license": "CC-BY-NC-SA-4.0", "keywords": [ "Chloris", "Biomass", "MODIS", "Carbon" ], "providers": [ { "url": "http://chloris.earth/", "name": "Chloris", "roles": [ "producer", "licensor" ] }, { "url": "https://planetarycomputer.microsoft.com", "name": "Microsoft", "roles": [ "host", "processor" ] } ], "summaries": { "gsd": [ 4633 ] }, "description": "The Chloris Global Biomass 2003 - 2019 dataset provides estimates of stock and change in aboveground biomass for Earth's terrestrial woody vegetation ecosystems. It covers the period 2003 - 2019, at annual time steps. The global dataset has a circa 4.6 km spatial resolution.\n\nThe maps and data sets were generated by combining multiple remote sensing measurements from space borne satellites, processed using state-of-the-art machine learning and statistical methods, validated with field data from multiple countries. The dataset provides direct estimates of aboveground stock and change, and are not based on land use or land cover area change, and as such they include gains and losses of carbon stock in all types of woody vegetation - whether natural or plantations.\n\nAnnual stocks are expressed in units of tons of biomass. Annual changes in stocks are expressed in units of CO2 equivalent, i.e., the amount of CO2 released from or taken up by terrestrial ecosystems for that specific pixel.\n\nThe spatial data sets are available on [Microsoft’s Planetary Computer](https://planetarycomputer.microsoft.com/dataset/chloris-biomass) under a Creative Common license of the type Attribution-Non Commercial-Share Alike [CC BY-NC-SA](https://spdx.org/licenses/CC-BY-NC-SA-4.0.html).\n\n[Chloris Geospatial](https://chloris.earth/) is a mission-driven technology company that develops software and data products on the state of natural capital for use by business, governments, and the social sector.\n", "item_assets": { "biomass": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "Annual estimates of aboveground woody biomass.", "raster:bands": [ { "unit": "tonnes", "nodata": 2147483647, "data_type": "uint32" } ] }, "biomass_wm": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "Annual estimates of aboveground woody biomass (Web Mercator).", "raster:bands": [ { "unit": "tonnes", "nodata": 2147483647, "data_type": "uint32" } ] }, "biomass_change": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "Annual estimates of changes (gains and losses) in aboveground woody biomass from the previous year.", "raster:bands": [ { "unit": "tonnes", "nodata": -32768, "data_type": "int16" } ] }, "biomass_change_wm": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "Annual estimates of changes (gains and losses) in aboveground woody biomass from the previous year (Web Mercator).", "raster:bands": [ { "unit": "tonnes", "nodata": -32768, "data_type": "int16" } ] } }, "stac_version": "1.0.0", "msft:container": "chloris-biomass", "stac_extensions": [ "https://stac-extensions.github.io/projection/v1.0.0/schema.json", "https://stac-extensions.github.io/raster/v1.1.0/schema.json" ], "msft:storage_account": "ai4edataeuwest", "msft:short_description": "The Chloris Global Biomass 2003 - 2019 dataset provides estimates of stock and change in aboveground biomass for Earth's terrestrial woody vegetation ecosystems during the period 2003 - 2019, at annual time steps. The global dataset has a circa 4.6 km spatial resolution." } ================================================ FILE: src/pypgstac/tests/data-files/hydration/collections/landsat-c2-l1.json ================================================ { "id": "landsat-c2-l1", "type": "Collection", "links": [ { "rel": "items", "type": "application/geo+json", "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/collections/landsat-c2-l1/items" }, { "rel": "parent", "type": "application/json", "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/" }, { "rel": "root", "type": "application/json", "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/" }, { "rel": "self", "type": "application/json", "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/collections/landsat-c2-l1" }, { "rel": "cite-as", "href": "https://doi.org/10.5066/P9AF14YV", "title": "Landsat 1-5 MSS Collection 2 Level-1" }, { "rel": "license", "href": "https://www.usgs.gov/core-science-systems/hdds/data-policy", "title": "Public Domain" }, { "rel": "describedby", "href": "https://planetarycomputer.microsoft.com/dataset/landsat-c2-l1", "title": "Human readable dataset overview and reference", "type": "text/html" } ], "title": "Landsat Collection 2 Level-1", "assets": { "thumbnail": { "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/landsat-c2-l1-thumb.png", "type": "image/png", "roles": ["thumbnail"], "title": "Landsat Collection 2 Level-1 thumbnail" } }, "extent": { "spatial": { "bbox": [[-180, -90, 180, 90]] }, "temporal": { "interval": [["1972-07-25T00:00:00Z", "2013-01-07T23:23:59Z"]] } }, "license": "proprietary", "keywords": ["Landsat", "USGS", "NASA", "Satellite", "Global", "Imagery"], "providers": [ { "url": "https://landsat.gsfc.nasa.gov/", "name": "NASA", "roles": ["producer", "licensor"] }, { "url": "https://www.usgs.gov/landsat-missions/landsat-collection-2-level-1-data", "name": "USGS", "roles": ["producer", "processor", "licensor"] }, { "url": "https://planetarycomputer.microsoft.com", "name": "Microsoft", "roles": ["host"] } ], "summaries": { "gsd": [79], "sci:doi": ["10.5066/P9AF14YV"], "eo:bands": [ { "name": "B4", "common_name": "green", "description": "Visible green (Landsat 1-3 Band B4)", "center_wavelength": 0.55, "full_width_half_max": 0.1 }, { "name": "B5", "common_name": "red", "description": "Visible red (Landsat 1-3 Band B5)", "center_wavelength": 0.65, "full_width_half_max": 0.1 }, { "name": "B6", "common_name": "nir08", "description": "Near infrared (Landsat 1-3 Band B6)", "center_wavelength": 0.75, "full_width_half_max": 0.1 }, { "name": "B7", "common_name": "nir09", "description": "Near infrared (Landsat 1-3 Band B7)", "center_wavelength": 0.95, "full_width_half_max": 0.3 }, { "name": "B1", "common_name": "green", "description": "Visible green (Landsat 4-5 Band B1)", "center_wavelength": 0.55, "full_width_half_max": 0.1 }, { "name": "B2", "common_name": "red", "description": "Visible red (Landsat 4-5 Band B2)", "center_wavelength": 0.65, "full_width_half_max": 0.1 }, { "name": "B3", "common_name": "nir08", "description": "Near infrared (Landsat 4-5 Band B3)", "center_wavelength": 0.75, "full_width_half_max": 0.1 }, { "name": "B4", "common_name": "nir09", "description": "Near infrared (Landsat 4-5 Band B4)", "center_wavelength": 0.95, "full_width_half_max": 0.3 } ], "platform": [ "landsat-1", "landsat-2", "landsat-3", "landsat-4", "landsat-5" ], "instruments": ["mss"], "view:off_nadir": [0] }, "description": "Landsat Collection 2 Level-1 data, consisting of quantized and calibrated scaled Digital Numbers (DN) representing the multispectral image data. These [Level-1](https://www.usgs.gov/landsat-missions/landsat-collection-2-level-1-data) data can be [rescaled](https://www.usgs.gov/landsat-missions/using-usgs-landsat-level-1-data-product) to top of atmosphere (TOA) reflectance and/or radiance. Thermal band data can be rescaled to TOA brightness temperature.\\n\\nThis dataset represents the global archive of Level-1 data from [Landsat Collection 2](https://www.usgs.gov/core-science-systems/nli/landsat/landsat-collection-2) acquired by the [Multispectral Scanner System](https://landsat.gsfc.nasa.gov/multispectral-scanner-system/) onboard Landsat 1 through Landsat 5 from July 7, 1972 to January 7, 2013. Images are stored in [cloud-optimized GeoTIFF](https://www.cogeo.org/) format.\\n", "item_assets": { "red": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "Red Band", "description": "Collection 2 Level-1 Red Band Top of Atmosphere Radiance", "raster:bands": [ { "unit": "watt/steradian/square_meter/micrometer", "nodata": 0, "data_type": "uint8", "spatial_resolution": 60 } ] }, "green": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "Green Band", "description": "Collection 2 Level-1 Green Band Top of Atmosphere Radiance", "raster:bands": [ { "unit": "watt/steradian/square_meter/micrometer", "nodata": 0, "data_type": "uint8", "spatial_resolution": 60 } ] }, "nir08": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "Near Infrared Band 0.8", "description": "Collection 2 Level-1 Near Infrared Band 0.8 Top of Atmosphere Radiance", "raster:bands": [ { "unit": "watt/steradian/square_meter/micrometer", "nodata": 0, "data_type": "uint8", "spatial_resolution": 60 } ] }, "nir09": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["data"], "title": "Near Infrared Band 0.9", "description": "Collection 2 Level-1 Near Infrared Band 0.9 Top of Atmosphere Radiance", "raster:bands": [ { "unit": "watt/steradian/square_meter/micrometer", "nodata": 0, "data_type": "uint8", "spatial_resolution": 60 } ] }, "mtl.txt": { "type": "text/plain", "roles": ["metadata"], "title": "Product Metadata File (txt)", "description": "Collection 2 Level-1 Product Metadata File (txt)" }, "mtl.xml": { "type": "application/xml", "roles": ["metadata"], "title": "Product Metadata File (xml)", "description": "Collection 2 Level-1 Product Metadata File (xml)" }, "mtl.json": { "type": "application/json", "roles": ["metadata"], "title": "Product Metadata File (json)", "description": "Collection 2 Level-1 Product Metadata File (json)" }, "qa_pixel": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["cloud"], "title": "Pixel Quality Assessment Band", "description": "Collection 2 Level-1 Pixel Quality Assessment Band", "raster:bands": [ { "unit": "bit index", "nodata": 1, "data_type": "uint16", "spatial_resolution": 60 } ] }, "qa_radsat": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": ["saturation"], "title": "Radiometric Saturation and Dropped Pixel Quality Assessment Band", "description": "Collection 2 Level-1 Radiometric Saturation and Dropped Pixel Quality Assessment Band", "raster:bands": [ { "unit": "bit index", "data_type": "uint16", "spatial_resolution": 60 } ] }, "thumbnail": { "type": "image/jpeg", "roles": ["thumbnail"], "title": "Thumbnail image" }, "reduced_resolution_browse": { "type": "image/jpeg", "roles": ["overview"], "title": "Reduced resolution browse image" } }, "stac_version": "1.0.0", "stac_extensions": [ "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json", "https://stac-extensions.github.io/view/v1.0.0/schema.json", "https://stac-extensions.github.io/scientific/v1.0.0/schema.json", "https://stac-extensions.github.io/raster/v1.0.0/schema.json", "https://stac-extensions.github.io/eo/v1.0.0/schema.json" ] } ================================================ FILE: src/pypgstac/tests/data-files/hydration/collections/sentinel-1-grd.json ================================================ { "id": "sentinel-1-grd", "type": "Collection", "links": [ { "rel": "items", "type": "application/geo+json", "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/collections/sentinel-1-grd/items" }, { "rel": "parent", "type": "application/json", "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/" }, { "rel": "root", "type": "application/json", "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/" }, { "rel": "self", "type": "application/json", "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/stac/collections/sentinel-1-grd" }, { "rel": "license", "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice", "title": "Copernicus Sentinel data terms" }, { "rel": "describedby", "href": "https://planetarycomputer.microsoft.com/dataset/sentinel-1-grd", "title": "Human readable dataset overview and reference", "type": "text/html" } ], "title": "Sentinel 1 Level-1 Ground Range Detected (GRD)", "assets": { "thumbnail": { "href": "https://ai4edatasetspublicassets.blob.core.windows.net/assets/pc_thumbnails/sentinel-1-grd.png", "type": "image/png", "roles": [ "thumbnail" ], "title": "Sentinel 1 GRD" } }, "extent": { "spatial": { "bbox": [ [ -180, -90, 180, 90 ] ] }, "temporal": { "interval": [ [ "2014-10-10T00:28:21Z", null ] ] } }, "license": "proprietary", "keywords": [ "ESA", "Copernicus", "Sentinel", "C-Band", "SAR", "GRD" ], "providers": [ { "url": "https://earth.esa.int/web/guest/home", "name": "ESA", "roles": [ "producer", "processor", "licensor" ] }, { "url": "https://planetarycomputer.microsoft.com", "name": "Microsoft", "roles": [ "host" ] } ], "summaries": { "platform": [ "SENTINEL-1A", "SENTINEL-1B" ], "constellation": [ "Sentinel-1" ], "s1:resolution": [ "full", "high", "medium" ], "s1:orbit_source": [ "DOWNLINK", "POEORB", "PREORB", "RESORB" ], "sar:looks_range": [ 5, 6, 3, 2 ], "sat:orbit_state": [ "ascending", "descending" ], "sar:product_type": [ "GRD" ], "sar:looks_azimuth": [ 1, 6, 2 ], "sar:polarizations": [ [ "VV", "VH" ], [ "HH", "HV" ], [ "VV" ], [ "VH" ], [ "HH" ], [ "HV" ] ], "sar:frequency_band": [ "C" ], "s1:processing_level": [ "1" ], "sar:instrument_mode": [ "IW", "EW", "SM" ], "sar:center_frequency": [ 5.405 ], "sar:resolution_range": [ 20, 23, 50, 93, 9 ], "s1:product_timeliness": [ "NRT-10m", "NRT-1h", "NRT-3h", "Fast-24h", "Off-line", "Reprocessing" ], "sar:resolution_azimuth": [ 22, 23, 50, 87, 9 ], "sar:pixel_spacing_range": [ 10, 25, 40, 3.5 ], "sar:observation_direction": [ "right" ], "sar:pixel_spacing_azimuth": [ 10, 25, 40, 3.5 ], "sar:looks_equivalent_number": [ 4.4, 29.7, 2.7, 10.7, 3.7 ], "sat:platform_international_designator": [ "2014-016A", "2016-025A", "0000-000A" ] }, "description": "...", "item_assets": { "hh": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "HH: horizontal transmit, horizontal receive", "description": "Amplitude of signal transmitted with horizontal polarization and received with horizontal polarization with radiometric terrain correction applied." }, "hv": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "HV: horizontal transmit, vertical receive", "description": "Amplitude of signal transmitted with horizontal polarization and received with vertical polarization with radiometric terrain correction applied." }, "vh": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "VH: vertical transmit, horizontal receive", "description": "Amplitude of signal transmitted with vertical polarization and received with horizontal polarization with radiometric terrain correction applied." }, "vv": { "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "VV: vertical transmit, vertical receive", "description": "Amplitude of signal transmitted with vertical polarization and received with vertical polarization with radiometric terrain correction applied." }, "thumbnail": { "type": "image/png", "roles": [ "thumbnail" ], "title": "Preview Image", "description": "An averaged, decimated preview image in PNG format. Single polarisation products are represented with a grey scale image. Dual polarisation products are represented by a single composite colour image in RGB with the red channel (R) representing the co-polarisation VV or HH), the green channel (G) represents the cross-polarisation (VH or HV) and the blue channel (B) represents the ratio of the cross an co-polarisations." }, "safe-manifest": { "type": "application/xml", "roles": [ "metadata" ], "title": "Manifest File", "description": "General product metadata in XML format. Contains a high-level textual description of the product and references to all of product's components, the product metadata, including the product identification and the resource references, and references to the physical location of each component file contained in the product." }, "schema-noise-hh": { "type": "application/xml", "roles": [ "metadata" ], "title": "Noise Schema", "description": "Estimated thermal noise look-up tables" }, "schema-noise-hv": { "type": "application/xml", "roles": [ "metadata" ], "title": "Noise Schema", "description": "Estimated thermal noise look-up tables" }, "schema-noise-vh": { "type": "application/xml", "roles": [ "metadata" ], "title": "Noise Schema", "description": "Estimated thermal noise look-up tables" }, "schema-noise-vv": { "type": "application/xml", "roles": [ "metadata" ], "title": "Noise Schema", "description": "Estimated thermal noise look-up tables" }, "schema-product-hh": { "type": "application/xml", "roles": [ "metadata" ], "title": "Product Schema", "description": "Describes the main characteristics corresponding to the band: state of the platform during acquisition, image properties, Doppler information, geographic location, etc." }, "schema-product-hv": { "type": "application/xml", "roles": [ "metadata" ], "title": "Product Schema", "description": "Describes the main characteristics corresponding to the band: state of the platform during acquisition, image properties, Doppler information, geographic location, etc." }, "schema-product-vh": { "type": "application/xml", "roles": [ "metadata" ], "title": "Product Schema", "description": "Describes the main characteristics corresponding to the band: state of the platform during acquisition, image properties, Doppler information, geographic location, etc." }, "schema-product-vv": { "type": "application/xml", "roles": [ "metadata" ], "title": "Product Schema", "description": "Describes the main characteristics corresponding to the band: state of the platform during acquisition, image properties, Doppler information, geographic location, etc." }, "schema-calibration-hh": { "type": "application/xml", "roles": [ "metadata" ], "title": "Calibration Schema", "description": "Calibration metadata including calibration information and the beta nought, sigma nought, gamma and digital number look-up tables that can be used for absolute product calibration." }, "schema-calibration-hv": { "type": "application/xml", "roles": [ "metadata" ], "title": "Calibration Schema", "description": "Calibration metadata including calibration information and the beta nought, sigma nought, gamma and digital number look-up tables that can be used for absolute product calibration." }, "schema-calibration-vh": { "type": "application/xml", "roles": [ "metadata" ], "title": "Calibration Schema", "description": "Calibration metadata including calibration information and the beta nought, sigma nought, gamma and digital number look-up tables that can be used for absolute product calibration." }, "schema-calibration-vv": { "type": "application/xml", "roles": [ "metadata" ], "title": "Calibration Schema", "description": "Calibration metadata including calibration information and the beta nought, sigma nought, gamma and digital number look-up tables that can be used for absolute product calibration." } }, "stac_version": "1.0.0", "msft:group_id": "sentinel-1", "msft:container": "s1-grd", "stac_extensions": [ "https://stac-extensions.github.io/sar/v1.0.0/schema.json", "https://stac-extensions.github.io/sat/v1.0.0/schema.json", "https://stac-extensions.github.io/item-assets/v1.0.0/schema.json" ], "msft:storage_account": "sentinel1euwest", "msft:short_description": "Sentinel-1 Level-1 Ground Range Detected (GRD) products consist of focused SAR data that has been detected, multi-looked and projected to ground range using an Earth ellipsoid model." } ================================================ FILE: src/pypgstac/tests/data-files/hydration/dehydrated-items/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2.json ================================================ { "id": "LM04_L1GS_001001_19830527_02_T2", "properties": { "platform": "landsat-4", "instruments": ["mss"], "created": "2022-04-17T03:44:26.179982Z", "gsd": 79, "description": "Landsat Collection 2 Level-1", "eo:cloud_cover": 32.0, "view:off_nadir": 0, "view:sun_elevation": 29.32047976, "view:sun_azimuth": 210.31823865, "proj:epsg": 32631, "proj:shape": [4308, 4371], "proj:transform": [60.0, 0.0, 378930.0, 0.0, -60.0, 9099030.0], "landsat:cloud_cover_land": 0.0, "landsat:wrs_type": "2", "landsat:wrs_path": "001", "landsat:wrs_row": "001", "landsat:collection_category": "T2", "landsat:collection_number": "02", "landsat:correction": "L1GS", "landsat:scene_id": "LM40010011983147KIS00", "sci:doi": "10.5066/P9AF14YV", "datetime": "1983-05-27T13:36:40.094000Z" }, "geometry": { "coordinates": [ [ [6.3329893, 81.9297399], [-3.9422024, 81.0681921], [1.4107147, 79.6375797], [10.5596079, 80.371579], [6.3329893, 81.9297399] ] ], "type": "Polygon" }, "links": [ { "rel": "cite-as", "href": "https://doi.org/10.5066/P9AF14YV", "title": "Landsat 1-5 MSS Collection 2 Level-1" }, { "rel": "via", "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l1/items/LM04_L1GS_001001_19830527_20210902_02_T2", "type": "application/json", "title": "USGS STAC Item" } ], "assets": { "thumbnail": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_thumb_small.jpeg" }, "reduced_resolution_browse": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_thumb_large.jpeg" }, "mtl.json": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_MTL.json" }, "mtl.txt": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_MTL.txt" }, "mtl.xml": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_MTL.xml" }, "qa_pixel": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_QA_PIXEL.TIF", "description": "Collection 2 Level-1 Pixel Quality Assessment Band (QA_PIXEL)", "classification:bitfields": [ { "name": "fill", "description": "Image or fill data", "offset": 0, "length": 1, "classes": [ { "value": 0, "name": "not_fill", "description": "Image data" }, { "value": 1, "name": "fill", "description": "Fill data" } ] }, { "name": "cloud", "description": "Cloud mask", "offset": 3, "length": 1, "classes": [ { "value": 0, "name": "not_cloud", "description": "Cloud confidence is not high" }, { "value": 1, "name": "cloud", "description": "High confidence cloud" } ] }, { "name": "cloud_confidence", "description": "Cloud confidence levels", "offset": 8, "length": 2, "classes": [ { "value": 0, "name": "not_set", "description": "No confidence level set" }, { "value": 1, "name": "low", "description": "Low confidence cloud" }, { "value": 2, "name": "reserved", "description": "Reserved - value not used" }, { "value": 3, "name": "high", "description": "High confidence cloud" } ] } ] }, "qa_radsat": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_QA_RADSAT.TIF", "description": "Collection 2 Level-1 Radiometric Saturation and Dropped Pixel Quality Assessment Band (QA_RADSAT)", "classification:bitfields": [ { "name": "band1", "description": "Band 1 radiometric saturation", "offset": 0, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 1 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 1 saturated" } ] }, { "name": "band2", "description": "Band 2 radiometric saturation", "offset": 1, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 2 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 2 saturated" } ] }, { "name": "band3", "description": "Band 3 radiometric saturation", "offset": 2, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 3 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 3 saturated" } ] }, { "name": "band4", "description": "Band 4 radiometric saturation", "offset": 3, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 4 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 4 saturated" } ] }, { "name": "band5", "description": "Band 5 radiometric saturation", "offset": 4, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 5 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 5 saturated" } ] }, { "name": "band6", "description": "Band 6 radiometric saturation", "offset": 5, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 6 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 6 saturated" } ] }, { "name": "band7", "description": "Band 7 radiometric saturation", "offset": 6, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 7 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 7 saturated" } ] }, { "name": "dropped", "description": "Dropped pixel", "offset": 9, "length": 1, "classes": [ { "value": 0, "name": "not_dropped", "description": "Detector has a value - pixel present" }, { "value": 1, "name": "dropped", "description": "Detector does not have a value - no data" } ] } ] }, "green": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_B1.TIF", "description": "Collection 2 Level-1 Green Band (B1) Top of Atmosphere Radiance", "eo:bands": [ { "name": "B1", "common_name": "green", "description": "Visible green", "center_wavelength": 0.55, "full_width_half_max": 0.1 } ], "raster:bands": [{ "scale": 0.8752, "offset": 2.9248 }] }, "red": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_B2.TIF", "eo:bands": [ { "name": "B2", "common_name": "red", "description": "Visible red", "center_wavelength": 0.65, "full_width_half_max": 0.1 } ], "raster:bands": [{ "scale": 0.62008, "offset": 3.07992 }] }, "nir08": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_B3.TIF", "description": "Collection 2 Level-1 Near Infrared Band 0.8 (B3) Top of Atmosphere Radiance", "eo:bands": [ { "name": "B3", "common_name": "nir08", "description": "Near infrared", "center_wavelength": 0.75, "full_width_half_max": 0.1 } ], "raster:bands": [{ "scale": 0.54921, "offset": 4.55079 }] }, "nir09": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_B4.TIF", "eo:bands": [ { "name": "B4", "common_name": "nir09", "description": "Near infrared", "center_wavelength": 0.95, "full_width_half_max": 0.3 } ], "raster:bands": [{ "unit": "𒍟※" }] } }, "bbox": [-4.69534637, 79.55635318, 11.95560413, 81.87586682], "stac_extensions": [ "https://stac-extensions.github.io/raster/v1.0.0/schema.json", "https://stac-extensions.github.io/eo/v1.0.0/schema.json", "https://stac-extensions.github.io/view/v1.0.0/schema.json", "https://stac-extensions.github.io/projection/v1.0.0/schema.json", "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json", "https://stac-extensions.github.io/classification/v1.0.0/schema.json", "https://stac-extensions.github.io/scientific/v1.0.0/schema.json" ] } ================================================ FILE: src/pypgstac/tests/data-files/hydration/raw-items/landsat-c2-l1/LM04_L1GS_001001_19830527_02_T2.json ================================================ { "type": "Feature", "stac_version": "1.0.0", "id": "LM04_L1GS_001001_19830527_02_T2", "properties": { "platform": "landsat-4", "instruments": ["mss"], "created": "2022-04-17T03:44:26.179982Z", "gsd": 79, "description": "Landsat Collection 2 Level-1", "eo:cloud_cover": 32.0, "view:off_nadir": 0, "view:sun_elevation": 29.32047976, "view:sun_azimuth": 210.31823865, "proj:epsg": 32631, "proj:shape": [4308, 4371], "proj:transform": [60.0, 0.0, 378930.0, 0.0, -60.0, 9099030.0], "landsat:cloud_cover_land": 0.0, "landsat:wrs_type": "2", "landsat:wrs_path": "001", "landsat:wrs_row": "001", "landsat:collection_category": "T2", "landsat:collection_number": "02", "landsat:correction": "L1GS", "landsat:scene_id": "LM40010011983147KIS00", "sci:doi": "10.5066/P9AF14YV", "datetime": "1983-05-27T13:36:40.094000Z" }, "geometry": { "coordinates": [ [ [6.3329893, 81.9297399], [-3.9422024, 81.0681921], [1.4107147, 79.6375797], [10.5596079, 80.371579], [6.3329893, 81.9297399] ] ], "type": "Polygon" }, "links": [ { "rel": "cite-as", "href": "https://doi.org/10.5066/P9AF14YV", "title": "Landsat 1-5 MSS Collection 2 Level-1" }, { "rel": "via", "href": "https://landsatlook.usgs.gov/stac-server/collections/landsat-c2l1/items/LM04_L1GS_001001_19830527_20210902_02_T2", "type": "application/json", "title": "USGS STAC Item" } ], "assets": { "thumbnail": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_thumb_small.jpeg", "type": "image/jpeg", "title": "Thumbnail image", "roles": ["thumbnail"] }, "reduced_resolution_browse": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_thumb_large.jpeg", "type": "image/jpeg", "title": "Reduced resolution browse image", "roles": ["overview"] }, "mtl.json": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_MTL.json", "type": "application/json", "title": "Product Metadata File (json)", "description": "Collection 2 Level-1 Product Metadata File (json)", "roles": ["metadata"] }, "mtl.txt": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_MTL.txt", "type": "text/plain", "title": "Product Metadata File (txt)", "description": "Collection 2 Level-1 Product Metadata File (txt)", "roles": ["metadata"] }, "mtl.xml": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_MTL.xml", "type": "application/xml", "title": "Product Metadata File (xml)", "description": "Collection 2 Level-1 Product Metadata File (xml)", "roles": ["metadata"] }, "qa_pixel": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_QA_PIXEL.TIF", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "title": "Pixel Quality Assessment Band", "description": "Collection 2 Level-1 Pixel Quality Assessment Band (QA_PIXEL)", "classification:bitfields": [ { "name": "fill", "description": "Image or fill data", "offset": 0, "length": 1, "classes": [ { "value": 0, "name": "not_fill", "description": "Image data" }, { "value": 1, "name": "fill", "description": "Fill data" } ] }, { "name": "cloud", "description": "Cloud mask", "offset": 3, "length": 1, "classes": [ { "value": 0, "name": "not_cloud", "description": "Cloud confidence is not high" }, { "value": 1, "name": "cloud", "description": "High confidence cloud" } ] }, { "name": "cloud_confidence", "description": "Cloud confidence levels", "offset": 8, "length": 2, "classes": [ { "value": 0, "name": "not_set", "description": "No confidence level set" }, { "value": 1, "name": "low", "description": "Low confidence cloud" }, { "value": 2, "name": "reserved", "description": "Reserved - value not used" }, { "value": 3, "name": "high", "description": "High confidence cloud" } ] } ], "raster:bands": [ { "nodata": 1, "data_type": "uint16", "spatial_resolution": 60, "unit": "bit index" } ], "roles": ["cloud"] }, "qa_radsat": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_QA_RADSAT.TIF", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "title": "Radiometric Saturation and Dropped Pixel Quality Assessment Band", "description": "Collection 2 Level-1 Radiometric Saturation and Dropped Pixel Quality Assessment Band (QA_RADSAT)", "classification:bitfields": [ { "name": "band1", "description": "Band 1 radiometric saturation", "offset": 0, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 1 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 1 saturated" } ] }, { "name": "band2", "description": "Band 2 radiometric saturation", "offset": 1, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 2 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 2 saturated" } ] }, { "name": "band3", "description": "Band 3 radiometric saturation", "offset": 2, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 3 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 3 saturated" } ] }, { "name": "band4", "description": "Band 4 radiometric saturation", "offset": 3, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 4 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 4 saturated" } ] }, { "name": "band5", "description": "Band 5 radiometric saturation", "offset": 4, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 5 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 5 saturated" } ] }, { "name": "band6", "description": "Band 6 radiometric saturation", "offset": 5, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 6 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 6 saturated" } ] }, { "name": "band7", "description": "Band 7 radiometric saturation", "offset": 6, "length": 1, "classes": [ { "value": 0, "name": "not_saturated", "description": "Band 7 not saturated" }, { "value": 1, "name": "saturated", "description": "Band 7 saturated" } ] }, { "name": "dropped", "description": "Dropped pixel", "offset": 9, "length": 1, "classes": [ { "value": 0, "name": "not_dropped", "description": "Detector has a value - pixel present" }, { "value": 1, "name": "dropped", "description": "Detector does not have a value - no data" } ] } ], "raster:bands": [ { "data_type": "uint16", "spatial_resolution": 60, "unit": "bit index" } ], "roles": ["saturation"] }, "green": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_B1.TIF", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "title": "Green Band", "description": "Collection 2 Level-1 Green Band (B1) Top of Atmosphere Radiance", "eo:bands": [ { "name": "B1", "common_name": "green", "description": "Visible green", "center_wavelength": 0.55, "full_width_half_max": 0.1 } ], "raster:bands": [ { "nodata": 0, "data_type": "uint8", "spatial_resolution": 60, "unit": "watt/steradian/square_meter/micrometer", "scale": 0.8752, "offset": 2.9248 } ], "roles": ["data"] }, "red": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_B2.TIF", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "title": "Red Band", "description": "Collection 2 Level-1 Red Band Top of Atmosphere Radiance", "eo:bands": [ { "name": "B2", "common_name": "red", "description": "Visible red", "center_wavelength": 0.65, "full_width_half_max": 0.1 } ], "raster:bands": [ { "nodata": 0, "data_type": "uint8", "spatial_resolution": 60, "unit": "watt/steradian/square_meter/micrometer", "scale": 0.62008, "offset": 3.07992 } ], "roles": ["data"] }, "nir08": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_B3.TIF", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "title": "Near Infrared Band 0.8", "description": "Collection 2 Level-1 Near Infrared Band 0.8 (B3) Top of Atmosphere Radiance", "eo:bands": [ { "name": "B3", "common_name": "nir08", "description": "Near infrared", "center_wavelength": 0.75, "full_width_half_max": 0.1 } ], "raster:bands": [ { "nodata": 0, "data_type": "uint8", "spatial_resolution": 60, "unit": "watt/steradian/square_meter/micrometer", "scale": 0.54921, "offset": 4.55079 } ], "roles": ["data"] }, "nir09": { "href": "https://landsateuwest.blob.core.windows.net/landsat-c2/level-1/standard/mss/1983/001/001/LM04_L1GS_001001_19830527_20210902_02_T2/LM04_L1GS_001001_19830527_20210902_02_T2_B4.TIF", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "title": "Near Infrared Band 0.9", "description": "Collection 2 Level-1 Near Infrared Band 0.9 Top of Atmosphere Radiance", "eo:bands": [ { "name": "B4", "common_name": "nir09", "description": "Near infrared", "center_wavelength": 0.95, "full_width_half_max": 0.3 } ], "raster:bands": [ { "nodata": 0, "data_type": "uint8", "spatial_resolution": 60 } ], "roles": ["data"] } }, "bbox": [-4.69534637, 79.55635318, 11.95560413, 81.87586682], "stac_extensions": [ "https://stac-extensions.github.io/raster/v1.0.0/schema.json", "https://stac-extensions.github.io/eo/v1.0.0/schema.json", "https://stac-extensions.github.io/view/v1.0.0/schema.json", "https://stac-extensions.github.io/projection/v1.0.0/schema.json", "https://landsat.usgs.gov/stac/landsat-extension/v1.1.1/schema.json", "https://stac-extensions.github.io/classification/v1.0.0/schema.json", "https://stac-extensions.github.io/scientific/v1.0.0/schema.json" ], "collection": "landsat-c2-l1" } ================================================ FILE: src/pypgstac/tests/data-files/hydration/raw-items/sentinel-1-grd/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C.json ================================================ { "id": "S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C", "bbox": [ 28.75231935, 1.1415594, 31.26726622, 3.11793585 ], "type": "Feature", "links": [ { "rel": "collection", "type": "application/json", "href": "https://planetarycomputer-test.microsoft.com/stac/collections/sentinel-1-grd" }, { "rel": "parent", "type": "application/json", "href": "https://planetarycomputer-test.microsoft.com/stac/collections/sentinel-1-grd" }, { "rel": "root", "type": "application/json", "href": "https://planetarycomputer-test.microsoft.com/stac/" }, { "rel": "self", "type": "application/geo+json", "href": "https://planetarycomputer-test.microsoft.com/stac/collections/sentinel-1-grd/items/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C" }, { "rel": "license", "href": "https://sentinel.esa.int/documents/247904/690755/Sentinel_Data_Legal_Notice" }, { "rel": "preview", "href": "https://pct-apis-staging.westeurope.cloudapp.azure.com/data/item/map?collection=sentinel-1-grd&item=S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C", "title": "Map of item", "type": "text/html" } ], "assets": { "vh": { "href": "https://sentinel1euwest.blob.core.windows.net/s1-grd/GRD/2022/4/28/IW/DV/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C_28ED/measurement/iw-vh.tiff", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "VH: vertical transmit, horizontal receive", "description": "Amplitude of signal transmitted with vertical polarization and received with horizontal polarization with radiometric terrain correction applied." }, "vv": { "href": "https://sentinel1euwest.blob.core.windows.net/s1-grd/GRD/2022/4/28/IW/DV/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C_28ED/measurement/iw-vv.tiff", "type": "image/tiff; application=geotiff; profile=cloud-optimized", "roles": [ "data" ], "title": "VV: vertical transmit, vertical receive", "description": "Amplitude of signal transmitted with vertical polarization and received with vertical polarization with radiometric terrain correction applied." }, "thumbnail": { "href": "https://sentinel1euwest.blob.core.windows.net/s1-grd/GRD/2022/4/28/IW/DV/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C_28ED/preview/quick-look.png", "type": "image/png", "roles": [ "thumbnail" ], "title": "Preview Image", "description": "An averaged, decimated preview image in PNG format. Single polarisation products are represented with a grey scale image. Dual polarisation products are represented by a single composite colour image in RGB with the red channel (R) representing the co-polarisation VV or HH), the green channel (G) represents the cross-polarisation (VH or HV) and the blue channel (B) represents the ratio of the cross an co-polarisations." }, "safe-manifest": { "href": "https://sentinel1euwest.blob.core.windows.net/s1-grd/GRD/2022/4/28/IW/DV/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C_28ED/manifest.safe", "type": "application/xml", "roles": [ "metadata" ], "title": "Manifest File", "description": "General product metadata in XML format. Contains a high-level textual description of the product and references to all of product's components, the product metadata, including the product identification and the resource references, and references to the physical location of each component file contained in the product." }, "schema-noise-vh": { "href": "https://sentinel1euwest.blob.core.windows.net/s1-grd/GRD/2022/4/28/IW/DV/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C_28ED/annotation/calibration/noise-iw-vh.xml", "type": "application/xml", "roles": [ "metadata" ], "title": "Noise Schema", "description": "Estimated thermal noise look-up tables" }, "schema-noise-vv": { "href": "https://sentinel1euwest.blob.core.windows.net/s1-grd/GRD/2022/4/28/IW/DV/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C_28ED/annotation/calibration/noise-iw-vv.xml", "type": "application/xml", "roles": [ "metadata" ], "title": "Noise Schema", "description": "Estimated thermal noise look-up tables" }, "schema-product-vh": { "href": "https://sentinel1euwest.blob.core.windows.net/s1-grd/GRD/2022/4/28/IW/DV/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C_28ED/annotation/rfi/rfi-iw-vh.xml", "type": "application/xml", "roles": [ "metadata" ], "title": "Product Schema", "description": "Describes the main characteristics corresponding to the band: state of the platform during acquisition, image properties, Doppler information, geographic location, etc." }, "schema-product-vv": { "href": "https://sentinel1euwest.blob.core.windows.net/s1-grd/GRD/2022/4/28/IW/DV/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C_28ED/annotation/rfi/rfi-iw-vv.xml", "type": "application/xml", "roles": [ "metadata" ], "title": "Product Schema", "description": "Describes the main characteristics corresponding to the band: state of the platform during acquisition, image properties, Doppler information, geographic location, etc." }, "schema-calibration-vh": { "href": "https://sentinel1euwest.blob.core.windows.net/s1-grd/GRD/2022/4/28/IW/DV/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C_28ED/annotation/calibration/calibration-iw-vh.xml", "type": "application/xml", "roles": [ "metadata" ], "title": "Calibration Schema", "description": "Calibration metadata including calibration information and the beta nought, sigma nought, gamma and digital number look-up tables that can be used for absolute product calibration." }, "schema-calibration-vv": { "href": "https://sentinel1euwest.blob.core.windows.net/s1-grd/GRD/2022/4/28/IW/DV/S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C_28ED/annotation/calibration/calibration-iw-vv.xml", "type": "application/xml", "roles": [ "metadata" ], "title": "Calibration Schema", "description": "Calibration metadata including calibration information and the beta nought, sigma nought, gamma and digital number look-up tables that can be used for absolute product calibration." } }, "geometry": { "type": "Polygon", "coordinates": [ [ [ 31.2672662, 2.6527481 ], [ 30.144719, 2.8900456 ], [ 29.0610893, 3.1179358 ], [ 28.948047, 2.5753315 ], [ 28.9109774, 2.3942923 ], [ 28.873316, 2.2133714 ], [ 28.8368458, 2.0321919 ], [ 28.801199, 1.8508305 ], [ 28.7523193, 1.6096632 ], [ 30.9486881, 1.1415594 ], [ 30.959178, 1.2021369 ], [ 30.9997364, 1.3830464 ], [ 31.0794822, 1.7451525 ], [ 31.1183746, 1.9264115 ], [ 31.1528986, 2.1085995 ], [ 31.1867151, 2.2909337 ], [ 31.2672662, 2.6527481 ] ] ] }, "collection": "sentinel-1-grd", "properties": { "datetime": "2022-04-28T03:44:30.207391Z", "platform": "SENTINEL-1A", "s1:shape": [ 25547, 16823 ], "end_datetime": "2022-04-28 03:44:42.707207+00:00", "constellation": "Sentinel-1", "s1:resolution": "high", "s1:datatake_id": "336188", "start_datetime": "2022-04-28 03:44:17.707575+00:00", "s1:orbit_source": "RESORB", "s1:slice_number": "5", "s1:total_slices": "14", "sar:looks_range": 5, "sat:orbit_state": "descending", "sar:product_type": "GRD", "sar:looks_azimuth": 1, "sar:polarizations": [ "VV", "VH" ], "sar:frequency_band": "C", "sat:absolute_orbit": 42968, "sat:relative_orbit": 21, "s1:processing_level": "1", "sar:instrument_mode": "IW", "sar:center_frequency": 5.405, "sar:resolution_range": 20, "s1:product_timeliness": "Fast-24h", "sar:resolution_azimuth": 22, "sar:pixel_spacing_range": 10, "sar:observation_direction": "right", "sar:pixel_spacing_azimuth": 10, "sar:looks_equivalent_number": 4.4, "s1:instrument_configuration_ID": "7", "sat:platform_international_designator": "2014-016A" }, "stac_extensions": [ "https://stac-extensions.github.io/sar/v1.0.0/schema.json", "https://stac-extensions.github.io/sat/v1.0.0/schema.json", "https://stac-extensions.github.io/eo/v1.0.0/schema.json" ], "stac_version": "1.0.0" } ================================================ FILE: src/pypgstac/tests/data-files/load/dehydrated.txt ================================================ chloris_biomass_50km_2017 0103000020E6100000010000000500000066666666667E66C0000000000080564066666666667E66C00000000000004EC066666666667E66400000000000004EC066666666667E6640000000000080564066666666667E66C00000000000805640 chloris-biomass 2016-07-31 00:00:00+00 2017-07-31 00:00:00+00 {"bbox": [-179.95, -60.0, 179.95, 90.0], "links": [], "assets": {"biomass": {"href": "https://ai4edataeuwest.blob.core.windows.net/chloris-biomass/cog/bio_2017.tif", "roles": "𒍟※", "file:size": 20568072}, "biomass_wm": {"href": "https://ai4edataeuwest.blob.core.windows.net/chloris-biomass/cog/bio_2017_merc.tif", "roles": "𒍟※", "file:size": 44219262, "proj:bbox": [-20037508.337301563, -8400523.19027713, 20034408.51494577, 149088735.41090125], "proj:shape": [33992, 8649], "proj:transform": [4633.12716525001, 0.0, -20037508.337301563, 0.0, -4633.127165250011, 149088735.41090125]}, "biomass_change": {"href": "https://ai4edataeuwest.blob.core.windows.net/chloris-biomass/cog/bio_change_2016-2017.tif", "roles": "𒍟※", "file:size": 11101605}, "biomass_change_wm": {"href": "https://ai4edataeuwest.blob.core.windows.net/chloris-biomass/cog/bio_change_2016-2017_merc.tif", "roles": "𒍟※", "file:size": 25997181, "proj:bbox": [-20037508.337301563, -8400523.19027713, 20034408.51494577, 149088735.41090125], "proj:shape": [33992, 8649], "proj:transform": [4633.12716525001, 0.0, -20037508.337301563, 0.0, -4633.127165250011, 149088735.41090125]}}, "properties": {"gsd": 4633, "datetime": "2017-01-01T00:00:00Z", "proj:bbox": [-20015109.354, -6671703.11790004, 20015109.35376009, 10007554.677], "proj:epsg": null, "proj:wkt2": "PROJCS[\\"unnamed\\",GEOGCS[\\"unnamed ellipse\\",DATUM[\\"unknown\\",SPHEROID[\\"unnamed\\",6371007.181,0]],PRIMEM[\\"Greenwich\\",0],UNIT[\\"degree\\",0.0174532925199433,AUTHORITY[\\"EPSG\\",\\"9122\\"]]],PROJECTION[\\"Sinusoidal\\"],PARAMETER[\\"longitude_of_center\\",0],PARAMETER[\\"false_easting\\",0],PARAMETER[\\"false_northing\\",0],UNIT[\\"metre\\",1,AUTHORITY[\\"EPSG\\",\\"9001\\"]],AXIS[\\"Easting\\",EAST],AXIS[\\"Northing\\",NORTH]]", "proj:shape": [3600, 8640], "end_datetime": "2017-07-31T00:00:00Z", "proj:transform": [4633.12716525001, 0.0, -20015109.354, 0.0, -4633.127165250011, 10007554.677], "start_datetime": "2016-07-31T00:00:00Z"}, "stac_extensions": ["https://stac-extensions.github.io/file/v2.0.0/schema.json", "https://stac-extensions.github.io/raster/v1.0.0/schema.json", "https://stac-extensions.github.io/projection/v1.0.0/schema.json"]} chloris_biomass_50km_2018 0103000020E6100000010000000500000066666666667E66C0000000000080564066666666667E66C00000000000004EC066666666667E66400000000000004EC066666666667E6640000000000080564066666666667E66C00000000000805640 chloris-biomass 2017-07-31 00:00:00+00 2018-07-31 00:00:00+00 {"bbox": [-179.95, -60.0, 179.95, 90.0], "links": [], "assets": {"biomass": {"href": "https://ai4edataeuwest.blob.core.windows.net/chloris-biomass/cog/bio_2018.tif", "roles": "𒍟※", "file:size": 20581566}, "biomass_wm": {"href": "https://ai4edataeuwest.blob.core.windows.net/chloris-biomass/cog/bio_2018_merc.tif", "roles": "𒍟※", "file:size": 44264008, "proj:bbox": [-20037508.337301563, -8400523.19027713, 20034408.51494577, 149088735.41090125], "proj:shape": [33992, 8649], "proj:transform": [4633.12716525001, 0.0, -20037508.337301563, 0.0, -4633.127165250011, 149088735.41090125]}, "biomass_change": {"href": "https://ai4edataeuwest.blob.core.windows.net/chloris-biomass/cog/bio_change_2017-2018.tif", "roles": "𒍟※", "file:size": 11278226}, "biomass_change_wm": {"href": "https://ai4edataeuwest.blob.core.windows.net/chloris-biomass/cog/bio_change_2017-2018_merc.tif", "roles": "𒍟※", "file:size": 26334351, "proj:bbox": [-20037508.337301563, -8400523.19027713, 20034408.51494577, 149088735.41090125], "proj:shape": [33992, 8649], "proj:transform": [4633.12716525001, 0.0, -20037508.337301563, 0.0, -4633.127165250011, 149088735.41090125]}}, "properties": {"gsd": 4633, "datetime": "2018-01-01T00:00:00Z", "proj:bbox": [-20015109.354, -6671703.11790004, 20015109.35376009, 10007554.677], "proj:epsg": null, "proj:wkt2": "PROJCS[\\"unnamed\\",GEOGCS[\\"unnamed ellipse\\",DATUM[\\"unknown\\",SPHEROID[\\"unnamed\\",6371007.181,0]],PRIMEM[\\"Greenwich\\",0],UNIT[\\"degree\\",0.0174532925199433,AUTHORITY[\\"EPSG\\",\\"9122\\"]]],PROJECTION[\\"Sinusoidal\\"],PARAMETER[\\"longitude_of_center\\",0],PARAMETER[\\"false_easting\\",0],PARAMETER[\\"false_northing\\",0],UNIT[\\"metre\\",1,AUTHORITY[\\"EPSG\\",\\"9001\\"]],AXIS[\\"Easting\\",EAST],AXIS[\\"Northing\\",NORTH]]", "proj:shape": [3600, 8640], "end_datetime": "2018-07-31T00:00:00Z", "proj:transform": [4633.12716525001, 0.0, -20015109.354, 0.0, -4633.127165250011, 10007554.677], "start_datetime": "2017-07-31T00:00:00Z"}, "stac_extensions": ["https://stac-extensions.github.io/file/v2.0.0/schema.json", "https://stac-extensions.github.io/raster/v1.0.0/schema.json", "https://stac-extensions.github.io/projection/v1.0.0/schema.json"]} ================================================ FILE: src/pypgstac/tests/data-files/queryables/test_queryables.json ================================================ { "$schema": "https://json-schema.org/draft/2019-09/schema", "$id": "https://example.com/stac/queryables", "type": "object", "title": "Test Queryables for PgSTAC", "description": "Test queryable names for PgSTAC", "properties": { "id": { "description": "Item identifier", "type": "string" }, "collection": { "description": "Collection identifier", "type": "string" }, "datetime": { "description": "Datetime", "type": "string", "format": "date-time" }, "geometry": { "description": "Geometry", "type": "object" }, "test:string_prop": { "description": "Test string property", "type": "string" }, "test:number_prop": { "description": "Test number property", "type": "number", "minimum": 0, "maximum": 100 }, "test:integer_prop": { "description": "Test integer property", "type": "integer" }, "test:datetime_prop": { "description": "Test datetime property", "type": "string", "format": "date-time" }, "test:array_prop": { "description": "Test array property", "type": "array", "items": { "type": "string" } } }, "additionalProperties": true } ================================================ FILE: src/pypgstac/tests/hydration/__init__.py ================================================ ================================================ FILE: src/pypgstac/tests/hydration/test_base_item.py ================================================ import json from pathlib import Path from typing import Any, Dict, cast from pypgstac.load import Loader HERE = Path(__file__).parent LANDSAT_COLLECTION = ( HERE / ".." / "data-files" / "hydration" / "collections" / "landsat-c2-l1.json" ) def test_landsat_c2_l1(loader: Loader) -> None: """Test that a base item is created when a collection is loaded and that it is equal to the item_assets of the collection . """ with open(LANDSAT_COLLECTION) as f: collection = json.load(f) loader.load_collections(str(LANDSAT_COLLECTION)) base_item = cast( Dict[str, Any], loader.db.query_one( "SELECT base_item FROM collections WHERE id=%s;", (collection["id"],), ), ) assert type(base_item) is dict assert base_item["collection"] == collection["id"] assert base_item["assets"] == collection["item_assets"] ================================================ FILE: src/pypgstac/tests/hydration/test_dehydrate.py ================================================ import json from pathlib import Path from typing import Any, Dict, cast from pypgstac import hydration from pypgstac.hydration import DO_NOT_MERGE_MARKER from pypgstac.load import Loader HERE = Path(__file__).parent LANDSAT_COLLECTION = ( HERE / ".." / "data-files" / "hydration" / "collections" / "landsat-c2-l1.json" ) LANDSAT_ITEM = ( HERE / ".." / "data-files" / "hydration" / "raw-items" / "landsat-c2-l1" / "LM04_L1GS_001001_19830527_02_T2.json" ) class TestDehydrate: def dehydrate( self, base_item: Dict[str, Any], item: Dict[str, Any], ) -> Dict[str, Any]: return hydration.dehydrate(base_item, item) def test_landsat_c2_l1(self, loader: Loader) -> None: """ Test that a dehydrated item is created properly from a raw item against a base item from a collection. """ with open(LANDSAT_COLLECTION) as f: collection = json.load(f) loader.load_collections(str(LANDSAT_COLLECTION)) with open(LANDSAT_ITEM) as f: item = json.load(f) base_item = cast( Dict[str, Any], loader.db.query_one( "SELECT base_item FROM collections WHERE id=%s;", (collection["id"],), ), ) assert type(base_item) is dict dehydrated = self.dehydrate(base_item, item) # Expect certain keys on base and not on dehydrated only_base_keys = ["type", "collection", "stac_version"] assert all(k in base_item for k in only_base_keys) assert not any(k in dehydrated for k in only_base_keys) # Expect certain keys on dehydrated and not on base only_dehydrated_keys = ["id", "bbox", "geometry", "properties"] assert not any(k in base_item for k in only_dehydrated_keys) assert all(k in dehydrated for k in only_dehydrated_keys) # Properties, links should be exactly the same pre- and post-dehydration assert item["properties"] == dehydrated["properties"] assert item["links"] == dehydrated["links"] # Check specific assets are dehydrated correctly thumbnail = dehydrated["assets"]["thumbnail"] assert list(thumbnail.keys()) == ["href"] assert thumbnail["href"] == item["assets"]["thumbnail"]["href"] # Red asset raster bands have additional `scale` and `offset` keys red = dehydrated["assets"]["red"] assert list(red.keys()) == ["href", "eo:bands", "raster:bands"] assert len(red["raster:bands"]) == 1 assert list(red["raster:bands"][0].keys()) == ["scale", "offset"] item_red_rb = item["assets"]["red"]["raster:bands"][0] assert red["raster:bands"] == [ {"scale": item_red_rb["scale"], "offset": item_red_rb["offset"]}, ] # nir09 asset raster bands does not have a `unit` attribute, which is # present on base nir09 = dehydrated["assets"]["nir09"] assert list(nir09.keys()) == ["href", "eo:bands", "raster:bands"] assert len(nir09["raster:bands"]) == 1 assert list(nir09["raster:bands"][0].keys()) == ["unit"] assert nir09["raster:bands"] == [{"unit": DO_NOT_MERGE_MARKER}] def test_single_depth_equals(self) -> None: base_item = {"a": "first", "b": "second", "c": "third"} item = {"a": "first", "b": "second", "c": "third"} dehydrated = self.dehydrate(base_item, item) assert dehydrated == {} def test_nested_equals(self) -> None: base_item = {"a": "first", "b": "second", "c": {"d": "third"}} item = {"a": "first", "b": "second", "c": {"d": "third"}} dehydrated = self.dehydrate(base_item, item) assert dehydrated == {} def test_nested_extra_keys(self) -> None: """ Test that items having nested dicts with keys not in base item preserve the additional keys in the dehydrated item. """ base_item = {"a": "first", "b": "second", "c": {"d": "third"}} item = { "a": "first", "b": "second", "c": {"d": "third", "e": "fourth", "f": "fifth"}, } dehydrated = self.dehydrate(base_item, item) assert dehydrated == {"c": {"e": "fourth", "f": "fifth"}} def test_list_of_dicts_extra_keys(self) -> None: """Test that an equal length list of dicts is dehydrated correctly.""" base_item = {"a": [{"b1": 1, "b2": 2}, {"c1": 1, "c2": 2}]} item = {"a": [{"b1": 1, "b2": 2, "b3": 3}, {"c1": 1, "c2": 2, "c3": 3}]} dehydrated = self.dehydrate(base_item, item) assert "a" in dehydrated assert dehydrated["a"] == [{"b3": 3}, {"c3": 3}] def test_equal_len_list_of_mixed_types(self) -> None: """ Test that a list of equal length containing matched types at each index dehydrates dicts and preserves item-values of other types. """ base_item = {"a": [{"b1": 1, "b2": 2}, "foo", {"c1": 1, "c2": 2}, "bar"]} item = { "a": [ {"b1": 1, "b2": 2, "b3": 3}, "far", {"c1": 1, "c2": 2, "c3": 3}, "boo", ], } dehydrated = self.dehydrate(base_item, item) assert "a" in dehydrated assert dehydrated["a"] == [{"b3": 3}, "far", {"c3": 3}, "boo"] def test_unequal_len_list(self) -> None: """Test that unequal length lists preserve the item value exactly.""" base_item = {"a": [{"b1": 1}, {"c1": 1}, {"d1": 1}]} item = {"a": [{"b1": 1, "b2": 2}, {"c1": 1, "c2": 2}]} dehydrated = self.dehydrate(base_item, item) assert "a" in dehydrated assert dehydrated["a"] == item["a"] def test_marked_non_merged_fields(self) -> None: base_item = {"a": "first", "b": "second", "c": {"d": "third", "e": "fourth"}} item = { "a": "first", "b": "second", "c": {"d": "third", "f": "fifth"}, } dehydrated = self.dehydrate(base_item, item) assert dehydrated == {"c": {"e": DO_NOT_MERGE_MARKER, "f": "fifth"}} def test_marked_non_merged_fields_in_list(self) -> None: base_item = { "a": [{"b": "first", "d": "third"}, {"c": "second", "e": "fourth"}], } item = {"a": [{"b": "first"}, {"c": "second", "f": "fifth"}]} dehydrated = self.dehydrate(base_item, item) assert dehydrated == { "a": [ {"d": DO_NOT_MERGE_MARKER}, {"e": DO_NOT_MERGE_MARKER, "f": "fifth"}, ], } def test_deeply_nested_dict(self) -> None: base_item = {"a": {"b": {"c": {"d": "first", "d1": "second"}}}} item = {"a": {"b": {"c": {"d": "first", "d1": "second", "d2": "third"}}}} dehydrated = self.dehydrate(base_item, item) assert dehydrated == {"a": {"b": {"c": {"d2": "third"}}}} def test_equal_list_of_non_dicts(self) -> None: """Values of lists that match base_item should be dehydrated off.""" base_item = {"assets": {"thumbnail": {"roles": ["thumbnail"]}}} item = { "assets": {"thumbnail": {"roles": ["thumbnail"], "href": "http://foo.com"}}, } dehydrated = self.dehydrate(base_item, item) assert dehydrated == {"assets": {"thumbnail": {"href": "http://foo.com"}}} def test_invalid_assets_marked(self) -> None: """ Assets can be included on item-assets that are not uniformly included on individual items. Ensure that base item asset keys without a matching item key are marked do-no-merge after dehydration. """ base_item = { "type": "Feature", "assets": { "asset1": {"name": "Asset one"}, "asset2": {"name": "Asset two"}, }, } hydrated = { "assets": {"asset1": {"name": "Asset one", "href": "http://foo.com"}}, } dehydrated = self.dehydrate(base_item, hydrated) assert dehydrated == { "type": DO_NOT_MERGE_MARKER, "assets": { "asset1": {"href": "http://foo.com"}, "asset2": DO_NOT_MERGE_MARKER, }, } def test_top_level_base_keys_marked(self) -> None: """ Top level keys on the base item not present on the incoming item should be marked as do not merge, no matter the nesting level. """ base_item = { "single": "Feature", "double": {"nested": "value"}, "triple": {"nested": {"deep": "value"}}, "included": "value", } hydrated = {"included": "value", "unique": "value"} dehydrated = self.dehydrate(base_item, hydrated) assert dehydrated == { "single": DO_NOT_MERGE_MARKER, "double": DO_NOT_MERGE_MARKER, "triple": DO_NOT_MERGE_MARKER, "unique": "value", } ================================================ FILE: src/pypgstac/tests/hydration/test_dehydrate_pg.py ================================================ import os from contextlib import contextmanager from typing import Any, Dict, Generator import psycopg from pypgstac.db import PgstacDB from pypgstac.migrate import Migrate from .test_dehydrate import TestDehydrate as TDehydrate class TestDehydratePG(TDehydrate): """Class to test Dehydration using pgstac.""" @contextmanager def db(self) -> Generator: """Set up database connection.""" origdb: str = os.getenv("PGDATABASE", "") with psycopg.connect(autocommit=True) as conn: try: conn.execute("CREATE DATABASE pgstactestdb;") except psycopg.errors.DuplicateDatabase: pass os.environ["PGDATABASE"] = "pgstactestdb" pgdb = PgstacDB() with psycopg.connect(autocommit=True) as conn: conn.execute("DROP SCHEMA IF EXISTS pgstac CASCADE;") Migrate(pgdb).run_migration() yield pgdb pgdb.close() os.environ["PGDATABASE"] = origdb def dehydrate( self, base_item: Dict[str, Any], item: Dict[str, Any], ) -> Dict[str, Any]: """Dehydrate item using pgstac.""" with self.db() as db: return next(db.func("strip_jsonb", item, base_item))[0] ================================================ FILE: src/pypgstac/tests/hydration/test_hydrate.py ================================================ """Test Hydration.""" import json from copy import deepcopy from pathlib import Path from typing import Any, Dict, cast from pypgstac import hydration from pypgstac.hydration import DO_NOT_MERGE_MARKER from pypgstac.load import Loader HERE = Path(__file__).parent LANDSAT_COLLECTION = ( HERE / ".." / "data-files" / "hydration" / "collections" / "landsat-c2-l1.json" ) LANDSAT_DEHYDRATED_ITEM = ( HERE / ".." / "data-files" / "hydration" / "dehydrated-items" / "landsat-c2-l1" / "LM04_L1GS_001001_19830527_02_T2.json" ) LANDSAT_ITEM = ( HERE / ".." / "data-files" / "hydration" / "raw-items" / "landsat-c2-l1" / "LM04_L1GS_001001_19830527_02_T2.json" ) class TestHydrate: def hydrate( self, base_item: Dict[str, Any], item: Dict[str, Any], ) -> Dict[str, Any]: hpy = hydration.hydrate_py(deepcopy(base_item), deepcopy(item)) hrs = hydration.hydrate(deepcopy(base_item), deepcopy(item)) assert hpy == hrs return hrs def test_landsat_c2_l1(self, loader: Loader) -> None: """Test that a dehydrated item is is equal to the raw item it was dehydrated from, against the base item of the collection . """ with open(LANDSAT_COLLECTION) as f: collection = json.load(f) loader.load_collections(str(LANDSAT_COLLECTION)) with open(LANDSAT_DEHYDRATED_ITEM) as f: dehydrated = json.load(f) with open(LANDSAT_ITEM) as f: raw_item = json.load(f) base_item = cast( Dict[str, Any], loader.db.query_one( "SELECT base_item FROM collections WHERE id=%s;", (collection["id"],), ), ) assert type(base_item) is dict hydrated = self.hydrate(base_item, dehydrated) assert hydrated == raw_item def test_full_hydrate(self) -> None: base_item = {"a": "first", "b": "second", "c": "third"} dehydrated: Dict[str, Any] = {} rehydrated = self.hydrate(base_item, dehydrated) assert rehydrated == base_item def test_full_nested(self) -> None: base_item = {"a": "first", "b": "second", "c": {"d": "third"}} dehydrated: Dict[str, Any] = {} rehydrated = self.hydrate(base_item, dehydrated) assert rehydrated == base_item def test_nested_extra_keys(self) -> None: """ Test that items having nested dicts with keys not in base item preserve the additional keys in the dehydrated item. """ base_item = {"a": "first", "b": "second", "c": {"d": "third"}} dehydrated = {"c": {"e": "fourth", "f": "fifth"}} hydrated = self.hydrate(base_item, dehydrated) assert hydrated == { "a": "first", "b": "second", "c": {"d": "third", "e": "fourth", "f": "fifth"}, } def test_list_of_dicts_extra_keys(self) -> None: """Test that an equal length list of dicts is hydrated correctly.""" base_item = {"a": [{"b1": 1, "b2": 2}, {"c1": 1, "c2": 2}]} dehydrated = {"a": [{"b3": 3}, {"c3": 3}]} hydrated = self.hydrate(base_item, dehydrated) assert hydrated == { "a": [{"b1": 1, "b2": 2, "b3": 3}, {"c1": 1, "c2": 2, "c3": 3}], } def test_equal_len_list_of_mixed_types(self) -> None: """ Test that a list of equal length containing matched types at each index dehydrates dicts and preserves item-values of other types. """ base_item = {"a": [{"b1": 1, "b2": 2}, "foo", {"c1": 1, "c2": 2}, "bar"]} dehydrated = {"a": [{"b3": 3}, "far", {"c3": 3}, "boo"]} hydrated = self.hydrate(base_item, dehydrated) assert hydrated == { "a": [ {"b1": 1, "b2": 2, "b3": 3}, "far", {"c1": 1, "c2": 2, "c3": 3}, "boo", ], } def test_unequal_len_list(self) -> None: """Test that unequal length lists preserve the item value exactly.""" base_item = {"a": [{"b1": 1}, {"c1": 1}, {"d1": 1}]} dehydrated = {"a": [{"b1": 1, "b2": 2}, {"c1": 1, "c2": 2}]} hydrated = self.hydrate(base_item, dehydrated) assert hydrated == dehydrated def test_marked_non_merged_fields(self) -> None: base_item = { "a": "first", "b": "second", "c": {"d": "third", "e": "fourth"}, } dehydrated = {"c": {"e": DO_NOT_MERGE_MARKER, "f": "fifth"}} hydrated = self.hydrate(base_item, dehydrated) assert hydrated == { "a": "first", "b": "second", "c": {"d": "third", "f": "fifth"}, } def test_marked_non_merged_fields_in_list(self) -> None: base_item = { "a": [{"b": "first", "d": "third"}, {"c": "second", "e": "fourth"}], } dehydrated = { "a": [ {"d": DO_NOT_MERGE_MARKER}, {"e": DO_NOT_MERGE_MARKER, "f": "fifth"}, ], } hydrated = self.hydrate(base_item, dehydrated) assert hydrated == {"a": [{"b": "first"}, {"c": "second", "f": "fifth"}]} def test_deeply_nested_dict(self) -> None: base_item = {"a": {"b": {"c": {"d": "first", "d1": "second"}}}} dehydrated = {"a": {"b": {"c": {"d2": "third"}}}} hydrated = self.hydrate(base_item, dehydrated) assert hydrated == { "a": {"b": {"c": {"d": "first", "d1": "second", "d2": "third"}}}, } def test_equal_list_of_non_dicts(self) -> None: """Values of lists that match base_item should be hydrated back on.""" base_item = {"assets": {"thumbnail": {"roles": ["thumbnail"]}}} dehydrated = {"assets": {"thumbnail": {"href": "http://foo.com"}}} hydrated = self.hydrate(base_item, dehydrated) assert hydrated == { "assets": {"thumbnail": {"roles": ["thumbnail"], "href": "http://foo.com"}}, } def test_invalid_assets_removed(self) -> None: """ Assets can be included on item-assets that are not uniformly included on individual items. Ensure that item asset keys from base_item aren't included after hydration. """ base_item = { "type": "Feature", "assets": { "asset1": {"name": "Asset one"}, "asset2": {"name": "Asset two"}, }, } dehydrated = { "assets": { "asset1": {"href": "http://foo.com"}, "asset2": DO_NOT_MERGE_MARKER, }, } hydrated = self.hydrate(base_item, dehydrated) assert hydrated == { "type": "Feature", "assets": {"asset1": {"name": "Asset one", "href": "http://foo.com"}}, } def test_top_level_base_keys_marked(self) -> None: """ Top level keys on the base item not present on the incoming item should be marked as do not merge, no matter the nesting level. """ base_item = { "single": "Feature", "double": {"nested": "value"}, "triple": {"nested": {"deep": "value"}}, "included": "value", } dehydrated = { "single": DO_NOT_MERGE_MARKER, "double": DO_NOT_MERGE_MARKER, "triple": DO_NOT_MERGE_MARKER, "unique": "value", } hydrated = self.hydrate(base_item, dehydrated) assert hydrated == {"included": "value", "unique": "value"} def test_base_none(self) -> None: base_item = {"value": None} dehydrated = {"value": {"a": "b"}} hydrated = self.hydrate(base_item, dehydrated) assert hydrated == {"value": {"a": "b"}} ================================================ FILE: src/pypgstac/tests/hydration/test_hydrate_pg.py ================================================ """Test Hydration in PgSTAC.""" import os from contextlib import contextmanager from typing import Any, Dict, Generator import psycopg from pypgstac.db import PgstacDB from pypgstac.migrate import Migrate from .test_hydrate import TestHydrate as THydrate class TestHydratePG(THydrate): """Test hydration using PgSTAC.""" @contextmanager def db(self) -> Generator[PgstacDB, None, None]: """Set up database.""" origdb: str = os.getenv("PGDATABASE", "") with psycopg.connect(autocommit=True) as conn: try: conn.execute("CREATE DATABASE pgstactestdb;") except psycopg.errors.DuplicateDatabase: pass os.environ["PGDATABASE"] = "pgstactestdb" pgdb = PgstacDB() with psycopg.connect(autocommit=True) as conn: conn.execute("DROP SCHEMA IF EXISTS pgstac CASCADE;") Migrate(pgdb).run_migration() yield pgdb pgdb.close() os.environ["PGDATABASE"] = origdb def hydrate( self, base_item: Dict[str, Any], item: Dict[str, Any], ) -> Dict[str, Any]: """Hydrate using pgstac.""" with self.db() as db: return next(db.func("content_hydrate", item, base_item))[0] ================================================ FILE: src/pypgstac/tests/test_benchmark.py ================================================ import json import uuid from datetime import datetime, timezone from math import ceil from typing import Any, Dict, Generator, Tuple import morecantile import psycopg import pytest from pypgstac.load import Loader, Methods XMIN, YMIN = 0, 0 AOI_WIDTH = 50 AOI_HEIGHT = 50 ITEM_WIDTHS = [0.5, 0.75, 1, 1.5, 2, 3, 4, 5, 6, 8, 10] TMS = morecantile.tms.get("WebMercatorQuad") def generate_items( item_size: Tuple[float, float], collection_id: str, ) -> Generator[Dict[str, Any], None, None]: item_width, item_height = item_size cols = ceil(AOI_WIDTH / item_width) rows = ceil(AOI_HEIGHT / item_height) # generate an item for each grid cell for row in range(rows): for col in range(cols): left = XMIN + (col * item_width) bottom = YMIN + (row * item_height) right = left + item_width top = bottom + item_height yield { "type": "Feature", "stac_version": "1.0.0", "id": str(uuid.uuid4()), "collection": collection_id, "geometry": { "type": "Polygon", "coordinates": [ [ [left, bottom], [right, bottom], [right, top], [left, top], [left, bottom], ], ], }, "bbox": [left, bottom, right, top], "properties": { "datetime": datetime.now(timezone.utc).isoformat(), }, } @pytest.fixture(scope="function") def search_hashes(loader: Loader) -> Dict[float, str]: search_hashes = {} for item_width in ITEM_WIDTHS: collection_id = f"collection-{str(item_width)}" collection = { "type": "Collection", "id": collection_id, "stac_version": "1.0.0", "description": f"Minimal test collection {collection_id}", "license": "proprietary", "extent": { "spatial": { "bbox": [XMIN, YMIN, XMIN + AOI_WIDTH, YMIN + AOI_HEIGHT], }, "temporal": { "interval": [[datetime.now(timezone.utc).isoformat(), None]], }, }, } loader.load_collections( iter([collection]), insert_mode=Methods.insert, ) loader.load_items( generate_items((item_width, item_width), collection_id), insert_mode=Methods.insert, ) with psycopg.connect(autocommit=True) as conn: with conn.cursor() as cursor: cursor.execute( "SELECT * FROM search_query(%s);", (json.dumps({"collections": [collection_id]}),), ) res = cursor.fetchone() assert res search_hashes[item_width] = res[0] return search_hashes @pytest.mark.benchmark( group="xyzsearch", min_rounds=3, warmup=True, warmup_iterations=2, ) @pytest.mark.parametrize("item_width", ITEM_WIDTHS) @pytest.mark.parametrize("zoom", range(3, 8 + 1)) def test1( benchmark, search_hashes: Dict[float, str], item_width: float, zoom: int, ) -> None: # get a tile from the center of the full AOI xmid = XMIN + AOI_WIDTH / 2 ymid = YMIN + AOI_HEIGHT / 2 tiles = TMS.tiles(xmid, ymid, xmid + 1, ymid + 1, [zoom]) tile = next(tiles) def xyzsearch_test(): with psycopg.connect(autocommit=True) as conn: with conn.cursor() as cursor: cursor.execute( "SELECT * FROM xyzsearch(%s, %s, %s, %s, %s, %s, %s, %s, %s, %s);", ( tile.x, tile.y, tile.z, search_hashes[item_width], json.dumps( { "include": ["assets", "id", "bbox", "collection"], }, ), # fields 100000, # scan_limit, 100000, # items limit "5 seconds", True, # exitwhenfull True, # skipcovered ), ) row = cursor.fetchone() assert row is not None _ = row[0] _ = benchmark(xyzsearch_test) ================================================ FILE: src/pypgstac/tests/test_load.py ================================================ """Tests for pypgstac.""" import json import re import threading from datetime import datetime, timezone from pathlib import Path from unittest import mock import pytest from psycopg.errors import UniqueViolation from version_parser import Version as V from pypgstac.db import PgstacDB from pypgstac.load import Loader, Methods, __version__, read_json HERE = Path(__file__).parent TEST_DATA_DIR = HERE.parent.parent / "pgstac" / "tests" / "testdata" TEST_COLLECTIONS_JSON = TEST_DATA_DIR / "collections.json" TEST_COLLECTIONS = TEST_DATA_DIR / "collections.ndjson" TEST_ITEMS = TEST_DATA_DIR / "items_private.ndjson" TEST_DEHYDRATED_ITEMS = TEST_DATA_DIR / "items.pgcopy" S1_GRD_COLLECTION = ( HERE / "data-files" / "hydration" / "collections" / "sentinel-1-grd.json" ) S1_GRD_ITEM = ( HERE / "data-files" / "hydration" / "raw-items" / "sentinel-1-grd" / "S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C.json" ) def version_increment(source_version: str) -> str: source_version = re.sub("-dev$", "", source_version) version = V(source_version) return ".".join( map( str, [ version.get_major_version(), version.get_minor_version(), version.get_patch_version() + 1, ], ), ) def test_load_collections_succeeds(loader: Loader) -> None: """Test pypgstac collections loader.""" loader.load_collections( str(TEST_COLLECTIONS), insert_mode=Methods.insert, ) def test_load_collections_json_succeeds(loader: Loader) -> None: """Test pypgstac collections loader.""" loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) def test_load_collections_json_duplicates_fails(loader: Loader) -> None: """Test pypgstac collections loader.""" loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) with pytest.raises(UniqueViolation): loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) def test_load_collections_json_duplicates_with_upsert(loader: Loader) -> None: """Test pypgstac collections loader.""" loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.upsert, ) def test_load_collections_json_duplicates_with_ignore(loader: Loader) -> None: """Test pypgstac collections loader.""" loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.ignore, ) def test_load_items_duplicates_fails(loader: Loader) -> None: """Test pypgstac collections loader.""" loader.load_collections( str(TEST_COLLECTIONS), insert_mode=Methods.insert, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) with pytest.raises(UniqueViolation): loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) def test_load_items_succeeds(loader: Loader) -> None: """Test pypgstac items loader.""" loader.load_collections( str(TEST_COLLECTIONS), insert_mode=Methods.upsert, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) def test_load_items_ignore_succeeds(loader: Loader) -> None: """Test pypgstac items ignore loader.""" loader.load_collections( str(TEST_COLLECTIONS), insert_mode=Methods.ignore, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.ignore, ) def test_load_items_upsert_succeeds(loader: Loader) -> None: """Test pypgstac items ignore loader.""" loader.load_collections( str(TEST_COLLECTIONS), insert_mode=Methods.ignore, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.upsert, ) def test_load_items_delsert_succeeds(loader: Loader) -> None: """Test pypgstac items ignore loader.""" loader.load_collections( str(TEST_COLLECTIONS), insert_mode=Methods.ignore, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.delsert, ) def test_partition_loads_default(loader: Loader) -> None: """Test pypgstac items ignore loader.""" loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.ignore, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) partitions = loader.db.query_one( """ SELECT count(*) from partitions; """, ) assert partitions == 1 def test_partition_loads_month(loader: Loader) -> None: """Test pypgstac items ignore loader.""" loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.ignore, ) if loader.db.connection is not None: loader.db.connection.execute( """ UPDATE collections SET partition_trunc='month'; """, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) partitions = loader.db.query_one( """ SELECT count(*) from partitions; """, ) assert partitions == 2 def test_partition_loads_year(loader: Loader) -> None: """Test pypgstac items ignore loader.""" loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.ignore, ) if loader.db.connection is not None: loader.db.connection.execute( """ UPDATE collections SET partition_trunc='year'; """, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) partitions = loader.db.query_one( """ SELECT count(*) from partitions; """, ) assert partitions == 1 def test_load_items_dehydrated_ignore_succeeds(loader: Loader) -> None: """Test pypgstac items ignore loader.""" loader.load_collections( str(TEST_COLLECTIONS), insert_mode=Methods.ignore, ) loader.load_items( str(TEST_DEHYDRATED_ITEMS), insert_mode=Methods.insert, dehydrated=True, ) loader.load_items( str(TEST_DEHYDRATED_ITEMS), insert_mode=Methods.ignore, dehydrated=True, ) def test_format_items_keys(loader: Loader) -> None: """Test pypgstac items ignore loader.""" loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.ignore, ) items_iter = read_json(str(TEST_ITEMS)) item_json = next(iter(items_iter)) out = loader.format_item(item_json) # Top level keys expected after format assert "id" in out assert "collection" in out assert "geometry" in out assert "content" in out assert "private" in out # Special keys expected not to be in the item content content_json = json.loads(out["content"]) assert "id" not in content_json assert "collection" not in content_json assert "geometry" not in content_json assert "private" not in content_json # Ensure bbox is included in content assert "bbox" in content_json def test_s1_grd_load_and_query(loader: Loader) -> None: """Test pypgstac items ignore loader.""" loader.load_collections( str(S1_GRD_COLLECTION), insert_mode=Methods.ignore, ) loader.load_items(str(S1_GRD_ITEM), insert_mode=Methods.insert) search_body = { "filter-lang": "cql2-json", "filter": { "op": "and", "args": [ { "op": "=", "args": [{"property": "collection"}, "sentinel-1-grd"], }, { "op": "=", "args": [ {"property": "id"}, "S1A_IW_GRDH_1SDV_20220428T034417_20220428T034442_042968_05213C", # noqa: E501 ], }, ], }, } res = next( loader.db.func( "search", search_body, ), )[0] res["features"][0] def test_load_dehydrated(loader: Loader) -> None: """Test loader for items dumped directly out of item table.""" collections = [ HERE / "data-files" / "hydration" / "collections" / "chloris-biomass.json", ] for collection in collections: loader.load_collections( str(collection), insert_mode=Methods.ignore, ) dehydrated_items = HERE / "data-files" / "load" / "dehydrated.txt" loader.load_items( str(dehydrated_items), insert_mode=Methods.insert, dehydrated=True, ) def test_load_collections_incompatible_version(loader: Loader) -> None: """Test pypgstac collections loader raises an exception for incompatible version.""" with mock.patch( "pypgstac.db.PgstacDB.version", new_callable=mock.PropertyMock, ) as mock_version: mock_version.return_value = "dummy" with pytest.raises(ValueError): loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) def test_load_items_incompatible_version(loader: Loader) -> None: """Test pypgstac items loader raises an exception for incompatible version.""" loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) with mock.patch( "pypgstac.db.PgstacDB.version", new_callable=mock.PropertyMock, ) as mock_version: mock_version.return_value = "dummy" with pytest.raises(ValueError): loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) def test_load_compatible_major_minor_version(loader: Loader) -> None: """Test pypgstac loader doesn't raise an exception.""" with mock.patch( "pypgstac.load.__version__", version_increment(__version__), ) as mock_version: loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) assert mock_version != loader.db.version def test_load_compatible_major_minor_version_with_dev_suffix(loader: Loader) -> None: """Test pypgstac loader accepts dev-suffixed library versions.""" with mock.patch( "pypgstac.load.__version__", f"{version_increment(__version__)}-dev", ): loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) def test_load_items_nopartitionconstraint_succeeds(loader: Loader) -> None: """Test pypgstac items loader.""" loader.load_collections( str(TEST_COLLECTIONS), insert_mode=Methods.upsert, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) cdtmin = loader.db.query_one( """ SELECT lower(constraint_dtrange)::text FROM partition_sys_meta WHERE partition = '_items_1'; """, ) assert cdtmin == "2011-07-31 00:00:00+00" with loader.db.connect() as conn: conn.execute( """ ALTER TABLE _items_1 DROP CONSTRAINT _items_1_dt; """, ) cdtmin = loader.db.query_one( """ SELECT lower(constraint_dtrange)::text FROM partition_sys_meta WHERE partition = '_items_1'; """, ) assert cdtmin == "-infinity" loader.load_items( str(TEST_ITEMS), insert_mode=Methods.upsert, ) cdtmin = loader.db.query_one( """ SELECT lower(constraint_dtrange)::text FROM partition_sys_meta WHERE partition = '_items_1'; """, ) assert cdtmin == "2011-07-31 00:00:00+00" def test_valid_srid(loader: Loader) -> None: """Test pypgstac items have a valid srid. https://github.com/stac-utils/pgstac/issues/357 """ loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.ignore, ) loader.load_items( str(TEST_ITEMS), insert_mode=Methods.insert, ) srid = loader.db.query_one( """ SELECT st_srid(geometry) from items LIMIT 1; """, ) assert isinstance(srid, int) assert srid > 0 def _make_item(item_id: str, collection: str, dt: str) -> dict: """Create a minimal STAC item with the given id, collection, and datetime.""" return { "id": item_id, "type": "Feature", "collection": collection, "geometry": { "type": "Polygon", "coordinates": [ [ [-85.31, 30.93], [-85.31, 31.00], [-85.38, 31.00], [-85.38, 30.93], [-85.31, 30.93], ], ], }, "bbox": [-85.38, 30.93, -85.31, 31.00], "links": [], "assets": {}, "properties": { "datetime": dt, }, "stac_version": "1.0.0", "stac_extensions": [], } def test_load_items_sequential_new_loader_per_item(db: PgstacDB) -> None: """Test that creating a new Loader per iteration with now() datetimes works. Reproduces a pattern where a for loop creates a fresh Loader for each iteration and loads a single item with datetime=now(). Each Loader has an empty _partition_cache, so it queries partition bounds from the DB each time. With slightly different datetimes, each iteration may trigger check_partition to drop and recreate constraints unnecessarily. """ # Load the collection once loader = Loader(db) loader.load_collections(str(TEST_COLLECTIONS), insert_mode=Methods.upsert) num_items = 10 collection_id = "pgstac-test-collection" for i in range(num_items): # Fresh loader each iteration — empty _partition_cache ldr = Loader(db) dt = datetime.now(timezone.utc).isoformat() item = _make_item(f"race-seq-{i}", collection_id, dt) ldr.load_items(iter([item]), insert_mode=Methods.upsert) count = db.query_one("SELECT count(*) FROM items;") assert count == num_items, ( f"Expected {num_items} items but found {count}. " "Sequential new-Loader-per-item with now() datetimes failed." ) def test_load_items_concurrent_new_loader_per_item(db: PgstacDB) -> None: """Test race condition with concurrent Loaders each loading one item. This replicates the scenario where multiple threads each instantiate a separate Loader and call load_items with a single item whose datetime is set to now(). Each Loader has its own _partition_cache, and the slightly different datetimes cause each to call check_partition, which drops and recreates partition constraints and refreshes materialized views. Concurrent execution triggers deadlocks, lock contention, and constraint violations. """ # Load the collection once loader = Loader(db) loader.load_collections(str(TEST_COLLECTIONS), insert_mode=Methods.upsert) num_items = 10 collection_id = "pgstac-test-collection" errors: list = [] def load_one_item(item_idx: int) -> None: try: ldr = Loader(PgstacDB()) dt = datetime.now(timezone.utc).isoformat() item = _make_item(f"race-concurrent-{item_idx}", collection_id, dt) ldr.load_items(iter([item]), insert_mode=Methods.upsert) except Exception as e: errors.append((item_idx, e)) threads = [] for i in range(num_items): t = threading.Thread(target=load_one_item, args=(i,)) threads.append(t) # Start all threads to maximize contention for t in threads: t.start() for t in threads: t.join(timeout=60) # Report any errors from threads if errors: error_msgs = [f"Item {idx}: {type(e).__name__}: {e}" for idx, e in errors] message = f"{len(errors)}/{num_items} concurrent loads failed:\n" + "\n".join( error_msgs, ) assert not errors, message count = db.query_one("SELECT count(*) FROM items;") assert count == num_items, ( f"Expected {num_items} items but found {count}. " "Concurrent new-Loader-per-item with now() datetimes lost items." ) ================================================ FILE: src/pypgstac/tests/test_queryables.py ================================================ """Tests for pypgstac queryables functionality.""" from pathlib import Path from unittest.mock import MagicMock, patch import pytest from pypgstac.db import PgstacDB from pypgstac.load import Loader, Methods from pypgstac.pypgstac import PgstacCLI HERE = Path(__file__).parent TEST_DATA_DIR = HERE.parent.parent / "pgstac" / "tests" / "testdata" TEST_COLLECTIONS_JSON = TEST_DATA_DIR / "collections.json" TEST_QUERYABLES_JSON = HERE / "data-files" / "queryables" / "test_queryables.json" def test_load_queryables_succeeds(db: PgstacDB) -> None: """Test pypgstac queryables loader.""" # Create a CLI instance cli = PgstacCLI(dsn=db.dsn) # Load the test queryables with index_fields specified for all fields cli.load_queryables( str(TEST_QUERYABLES_JSON), index_fields=[ "test:string_prop", "test:number_prop", "test:integer_prop", "test:datetime_prop", "test:array_prop", ], ) # Verify that the queryables were loaded result = db.query( """ SELECT name, property_wrapper, property_index_type FROM queryables WHERE name LIKE 'test:%' ORDER BY name; """, ) # Convert result to a list of dictionaries for easier assertion queryables = [ {"name": row[0], "property_wrapper": row[1], "property_index_type": row[2]} for row in result ] # Check that all test properties were loaded with correct wrappers assert len(queryables) == 5 # Check string property string_prop = next(q for q in queryables if q["name"] == "test:string_prop") assert string_prop["property_wrapper"] == "to_text" assert string_prop["property_index_type"] == "BTREE" # Check number property number_prop = next(q for q in queryables if q["name"] == "test:number_prop") assert number_prop["property_wrapper"] == "to_float" assert number_prop["property_index_type"] == "BTREE" # Check integer property integer_prop = next(q for q in queryables if q["name"] == "test:integer_prop") assert integer_prop["property_wrapper"] == "to_int" assert integer_prop["property_index_type"] == "BTREE" # Check datetime property datetime_prop = next(q for q in queryables if q["name"] == "test:datetime_prop") assert datetime_prop["property_wrapper"] == "to_tstz" assert datetime_prop["property_index_type"] == "BTREE" # Check array property array_prop = next(q for q in queryables if q["name"] == "test:array_prop") assert array_prop["property_wrapper"] == "to_text_array" assert array_prop["property_index_type"] == "BTREE" def test_load_queryables_without_index_fields(db: PgstacDB) -> None: """Test pypgstac queryables loader without index_fields parameter.""" # Create a CLI instance cli = PgstacCLI(dsn=db.dsn) # Load the test queryables without index_fields cli.load_queryables(str(TEST_QUERYABLES_JSON)) # Verify that the queryables were loaded without indexes result = db.query( """ SELECT name, property_wrapper, property_index_type FROM queryables WHERE name LIKE 'test:%' ORDER BY name; """, ) # Convert result to a list of dictionaries for easier assertion queryables = [ {"name": row[0], "property_wrapper": row[1], "property_index_type": row[2]} for row in result ] # Check that all test properties were loaded with correct wrappers but no indexes assert len(queryables) == 5 # Check that none of the properties have indexes for q in queryables: assert q["property_index_type"] is None def test_load_queryables_with_specific_index_fields(db: PgstacDB) -> None: """Test pypgstac queryables loader with specific index_fields.""" # Create a CLI instance cli = PgstacCLI(dsn=db.dsn) # Load the test queryables with only specific index_fields cli.load_queryables( str(TEST_QUERYABLES_JSON), index_fields=["test:string_prop", "test:datetime_prop"], ) # Verify that only the specified fields have indexes result = db.query( """ SELECT name, property_wrapper, property_index_type FROM queryables WHERE name LIKE 'test:%' ORDER BY name; """, ) # Convert result to a list of dictionaries for easier assertion queryables = [ {"name": row[0], "property_wrapper": row[1], "property_index_type": row[2]} for row in result ] # Check that all properties are loaded assert len(queryables) == 5 # Check that only the specified fields have indexes for q in queryables: if q["name"] in ["test:string_prop", "test:datetime_prop"]: assert q["property_index_type"] == "BTREE" else: assert q["property_index_type"] is None def test_load_queryables_empty_index_fields(db: PgstacDB) -> None: """Test pypgstac queryables loader with empty index_fields.""" # Create a CLI instance cli = PgstacCLI(dsn=db.dsn) # Load the test queryables with empty index_fields cli.load_queryables( str(TEST_QUERYABLES_JSON), index_fields=[], ) # Verify that no fields have indexes result = db.query( """ SELECT name, property_wrapper, property_index_type FROM queryables WHERE name LIKE 'test:%' ORDER BY name; """, ) # Convert result to a list of dictionaries for easier assertion queryables = [ {"name": row[0], "property_wrapper": row[1], "property_index_type": row[2]} for row in result ] # Check that no fields have indexes for q in queryables: assert q["property_index_type"] is None @patch("pypgstac.pypgstac.PgstacDB.connect") def test_maintain_partitions_called_only_with_index_fields(mock_connect): """Test that maintain_partitions is only called when index_fields is provided.""" # Mock the database connection mock_conn = MagicMock() mock_connect.return_value = mock_conn # Mock cursor mock_cursor = MagicMock() mock_conn.cursor.return_value.__enter__.return_value = mock_cursor # Create a CLI instance with the mocked connection cli = PgstacCLI(dsn="mock_dsn") # Create a temporary file with test queryables test_file = HERE / "data-files" / "queryables" / "temp_test.json" with open(test_file, "w") as f: f.write( """ { "type": "object", "title": "Test Properties", "properties": { "test:prop1": { "type": "string", "title": "Test Property 1" }, "test:prop2": { "type": "integer", "title": "Test Property 2" } } } """, ) # Case 1: With index_fields cli.load_queryables( str(test_file), index_fields=["test:prop1"], ) # Check that maintain_partitions was called maintain_calls = [ call_args for call_args in mock_cursor.execute.call_args_list if "maintain_partitions" in str(call_args) ] assert len(maintain_calls) == 1 # Reset mock mock_cursor.reset_mock() # Case 2: Without index_fields cli.load_queryables(str(test_file)) # Check that maintain_partitions was not called maintain_calls = [ call_args for call_args in mock_cursor.execute.call_args_list if "maintain_partitions" in str(call_args) ] assert len(maintain_calls) == 0 # Clean up test_file.unlink() def test_load_queryables_with_collections(db: PgstacDB, loader: Loader) -> None: """Test pypgstac queryables loader with specific collections.""" # Load test collections first loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) # Get collection IDs from the database result = db.query("SELECT id FROM collections LIMIT 2;") collection_ids = [row[0] for row in result] # Create a CLI instance cli = PgstacCLI(dsn=db.dsn) # Load queryables for specific collections cli.load_queryables( str(TEST_QUERYABLES_JSON), collection_ids=collection_ids, index_fields=["test:string_prop"], ) # Verify that the queryables were loaded with the correct collection IDs result = db.query( """ SELECT name, collection_ids, property_index_type FROM queryables WHERE name LIKE 'test:%' ORDER BY name; """, ) # Convert result to a list of dictionaries for easier assertion queryables = [ {"name": row[0], "collection_ids": row[1], "property_index_type": row[2]} for row in result ] # Check that all queryables have the correct collection IDs assert len(queryables) == 5 for q in queryables: assert set(q["collection_ids"]) == set(collection_ids) # Check that only test:string_prop has an index if q["name"] == "test:string_prop": assert q["property_index_type"] == "BTREE" else: assert q["property_index_type"] is None def test_load_queryables_update(db: PgstacDB) -> None: """Test updating existing queryables.""" # Create a CLI instance cli = PgstacCLI(dsn=db.dsn) # Load the test queryables with an index on number_prop cli.load_queryables(str(TEST_QUERYABLES_JSON), index_fields=["test:number_prop"]) # Modify the test queryables file to change property wrappers # This is simulated by directly updating the database db.query( """ UPDATE queryables SET property_wrapper = 'to_text' WHERE name = 'test:number_prop'; """, ) # Load the queryables again, but with a different index field cli.load_queryables(str(TEST_QUERYABLES_JSON), index_fields=["test:string_prop"]) # Verify that the property wrapper was updated and index changed result = db.query( """ SELECT name, property_wrapper, property_index_type FROM queryables WHERE name in ('test:number_prop', 'test:string_prop'); """, ) # Convert result to a list of dictionaries for easier assertion queryables = [ {"name": row[0], "property_wrapper": row[1], "property_index_type": row[2]} for row in result ] # Find the properties number_prop = next(q for q in queryables if q["name"] == "test:number_prop") string_prop = next(q for q in queryables if q["name"] == "test:string_prop") # The property wrapper should be back to to_float assert number_prop["property_wrapper"] == "to_float" # The index should be removed from number_prop assert number_prop["property_index_type"] is None # The index should be added to string_prop assert string_prop["property_index_type"] == "BTREE" def test_load_queryables_invalid_json(db: PgstacDB) -> None: """Test loading queryables with invalid JSON.""" # Create a CLI instance cli = PgstacCLI(dsn=db.dsn) # Create a temporary file with invalid JSON invalid_json_file = HERE / "data-files" / "queryables" / "invalid.json" with open(invalid_json_file, "w") as f: f.write("{") # Loading should raise an exception with pytest.raises((ValueError, SyntaxError)): cli.load_queryables(str(invalid_json_file)) # Clean up invalid_json_file.unlink() def test_load_queryables_delete_missing(db: PgstacDB) -> None: """Test loading queryables with delete_missing=True.""" # Create a CLI instance cli = PgstacCLI(dsn=db.dsn) # First, load the test queryables with indexes on all fields cli.load_queryables( str(TEST_QUERYABLES_JSON), index_fields=[ "test:string_prop", "test:number_prop", "test:integer_prop", "test:datetime_prop", "test:array_prop", ], ) # Create a temporary file with only one property partial_props_file = HERE / "data-files" / "queryables" / "partial_props.json" with open(partial_props_file, "w") as f: f.write( """ { "type": "object", "title": "Partial Properties", "properties": { "test:string_prop": { "type": "string", "title": "String Property" } } } """, ) # Load the partial queryables with delete_missing=True and index the string property cli.load_queryables( str(partial_props_file), delete_missing=True, index_fields=["test:string_prop"], ) # Verify that only the string property remains and has an index result = db.query( """ SELECT name, property_index_type FROM queryables WHERE name LIKE 'test:%' ORDER BY name; """, ) # Convert result to a list of dictionaries queryables = [{"name": row[0], "property_index_type": row[1]} for row in result] # Check that only the string property remains and has an index assert len(queryables) == 1 assert queryables[0]["name"] == "test:string_prop" assert queryables[0]["property_index_type"] == "BTREE" # Clean up partial_props_file.unlink() def test_load_queryables_delete_missing_with_collections( db: PgstacDB, loader: Loader, ) -> None: """Test loading queryables with delete_missing=True and specific collections.""" # Load test collections first loader.load_collections( str(TEST_COLLECTIONS_JSON), insert_mode=Methods.insert, ) # Get collection IDs from the database result = db.query("SELECT id FROM collections LIMIT 2;") collection_ids = [row[0] for row in result] # Create a CLI instance cli = PgstacCLI(dsn=db.dsn) # First, load all test queryables for the specific collections with indexes cli.load_queryables( str(TEST_QUERYABLES_JSON), collection_ids=collection_ids, index_fields=[ "test:string_prop", "test:number_prop", "test:integer_prop", "test:datetime_prop", "test:array_prop", ], ) # Create a temporary file with only one property partial_props_file = HERE / "data-files" / "queryables" / "partial_props.json" with open(partial_props_file, "w") as f: f.write( """ { "type": "object", "title": "Partial Properties", "properties": { "test:string_prop": { "type": "string", "title": "String Property" } } } """, ) # Load the partial queryables with delete_missing=True for the specific collections # but without an index cli.load_queryables( str(partial_props_file), collection_ids=collection_ids, delete_missing=True, ) # Verify that only the string property remains for the specific collections # and that it doesn't have an index result = db.query( """ SELECT name, collection_ids, property_index_type FROM queryables WHERE name LIKE 'test:%' ORDER BY name; """, ) # Convert result to a list of dictionaries queryables = [ {"name": row[0], "collection_ids": row[1], "property_index_type": row[2]} for row in result ] # Filter queryables for the specific collections specific_queryables = [ q for q in queryables if q["collection_ids"] and set(q["collection_ids"]) == set(collection_ids) ] # Check that only the string property remains for the specific collections assert len(specific_queryables) == 1 assert specific_queryables[0]["name"] == "test:string_prop" # Verify it doesn't have an index assert specific_queryables[0]["property_index_type"] is None # Clean up partial_props_file.unlink() def test_load_queryables_no_properties(db: PgstacDB) -> None: """Test loading queryables with no properties.""" # Create a CLI instance cli = PgstacCLI(dsn=db.dsn) # Create a temporary file with no properties no_props_file = HERE / "data-files" / "queryables" / "no_props.json" with open(no_props_file, "w") as f: f.write('{"type": "object", "title": "No Properties"}') # Loading should raise a ValueError with pytest.raises( ValueError, match="No properties found in queryables definition", ): cli.load_queryables(str(no_props_file)) # Clean up no_props_file.unlink()